Chapter 10: Writing Software for the NPU#

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

import tensorflow as tf

# Load your trained model
model = tf.keras.models.load_model('model.h5')

# Create converter
converter = tf.lite.TFLiteConverter.from_keras_model(model)

# Enable INT8 quantization
converter.optimizations = [tf.lite.Optimize.DEFAULT]

# Provide representative dataset for calibration
def representative_dataset():
    for _ in range(100):
        # Yield sample data matching your input shape
        yield [np.random.randn(1, 224, 224, 3).astype(np.float32)]

converter.representative_dataset = representative_dataset

# Set input/output to INT8
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.int8
converter.inference_output_type = tf.int8

# Convert model
tflite_model = converter.convert()

# Save quantized model
with open('model_quantized.tflite', 'wb') as f:
    f.write(tflite_model)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

from onnxruntime.quantization import quantize_dynamic, QuantType

# Dynamic quantization (weights only)
quantize_dynamic(
    model_input='model.onnx',
    model_output='model_int8.onnx',
    weight_type=QuantType.QInt8
)

# Static quantization with calibration
from onnxruntime.quantization import quantize_static, CalibrationDataReader

class DataReader(CalibrationDataReader):
    def __init__(self, calibration_data):
        self.data = calibration_data
        self.index = 0

    def get_next(self):
        if self.index < len(self.data):
            result = {'input': self.data[self.index]}
            self.index += 1
            return result
        return None

# Perform static quantization
quantize_static(
    model_input='model.onnx',
    model_output='model_static_int8.onnx',
    calibration_data_reader=DataReader(calibration_samples)
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

import tensorflow_model_optimization as tfmot

# Define pruning schedule
pruning_params = {
    'pruning_schedule': tfmot.sparsity.keras.PolynomialDecay(
        initial_sparsity=0.0,
        final_sparsity=0.5,
        begin_step=0,
        end_step=1000
    )
}

# Apply pruning to model
model_for_pruning = tfmot.sparsity.keras.prune_low_magnitude(
    model, **pruning_params
)

# Compile and train
model_for_pruning.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

# Add pruning callbacks
callbacks = [
    tfmot.sparsity.keras.UpdatePruningStep(),
    tfmot.sparsity.keras.PruningSummaries(log_dir='logs')
]

model_for_pruning.fit(
    train_data, train_labels,
    epochs=10,
    callbacks=callbacks
)

# Strip pruning wrappers and export
model_for_export = tfmot.sparsity.keras.strip_pruning(model_for_pruning)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 1. Prune the model
pruned_model = apply_pruning(original_model)

# 2. Fine-tune pruned model
pruned_model.fit(train_data, epochs=5)

# 3. Quantize the pruned model
converter = tf.lite.TFLiteConverter.from_keras_model(pruned_model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_dataset

tflite_model = converter.convert()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

// Note: This is conceptual - actual API may differ
var AIRuntime = require("@brightsign/ai-runtime");
var inference = new AIRuntime();

// Load model
inference.loadModel({
    modelPath: "/storage/models/object_detector.tflite",
    accelerator: "npu",
    threads: 1
}).then(function(model) {
    console.log("Model loaded successfully");
    // Model ready for inference
}).catch(function(error) {
    console.log("Failed to load model: " + error);
});

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

// Conceptual preprocessing for video frame
function preprocessFrame(videoFrame, inputWidth, inputHeight) {
    // 1. Resize frame to model input size
    var resized = resizeImage(videoFrame, inputWidth, inputHeight);

    // 2. Normalize pixel values (model dependent)
    // Common: scale to [0, 1] or [-1, 1]
    var normalized = normalizePixels(resized,
        mean=[127.5, 127.5, 127.5],
        std=[127.5, 127.5, 127.5]
    );

    // 3. Convert to required format (RGB vs BGR)
    var formatted = convertColorSpace(normalized, "RGB");

    // 4. Convert to INT8 if required by quantized model
    var quantized = floatToInt8(formatted);

    return quantized;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

// Conceptual audio preprocessing for speech recognition
function preprocessAudio(audioBuffer, sampleRate) {
    // 1. Resample to model's expected sample rate (e.g., 16kHz)
    var resampled = resampleAudio(audioBuffer, sampleRate, 16000);

    // 2. Extract features (MFCC, spectrogram, etc.)
    var features = extractMFCC(resampled, {
        numMfccCoeffs: 13,
        frameLength: 512,
        frameStep: 160
    });

    // 3. Normalize features
    var normalized = normalizeFeatures(features);

    return normalized;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

// Conceptual inference execution
function runInference(model, inputData) {
    return model.run({
        input: inputData
    }).then(function(results) {
        return results;
    });
}

// Example usage
var preprocessed = preprocessFrame(videoFrame, 224, 224);
runInference(model, preprocessed).then(function(output) {
    var predictions = output.predictions;
    console.log("Inference complete:", predictions);
});

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

// Conceptual streaming inference from video input
var VideoInputClass = require("@brightsign/videoinput");
var videoInput = new VideoInputClass();

// Set up frame capture
function processVideoStream(model) {
    videoInput.addEventListener('frame', function(frameEvent) {
        var frame = frameEvent.data;

        // Preprocess
        var input = preprocessFrame(frame, 224, 224);

        // Run inference
        runInference(model, input).then(function(results) {
            handleInferenceResults(results);
        });
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

function postprocessDetections(output, confidenceThreshold, iouThreshold) {
    var boxes = output.boxes;          // Bounding box coordinates
    var scores = output.scores;        // Confidence scores
    var classes = output.classes;      // Class IDs

    // Filter by confidence
    var filtered = [];
    for (var i = 0; i < scores.length; i++) {
        if (scores[i] > confidenceThreshold) {
            filtered.push({
                box: boxes[i],
                score: scores[i],
                class: classes[i],
                label: classLabels[classes[i]]
            });
        }
    }

    // Apply Non-Maximum Suppression
    var final = applyNMS(filtered, iouThreshold);

    return final;
}

// Usage
var detections = postprocessDetections(
    modelOutput,
    confidenceThreshold=0.5,
    iouThreshold=0.4
);

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

function postprocessClassification(output, topK) {
    var probabilities = softmax(output.logits);

    // Get top K predictions
    var indexed = probabilities.map(function(prob, idx) {
        return {score: prob, classId: idx};
    });

    indexed.sort(function(a, b) {
        return b.score - a.score;
    });

    return indexed.slice(0, topK).map(function(item) {
        return {
            label: classLabels[item.classId],
            confidence: item.score
        };
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

// Conceptual object detection implementation
function detectObjects(videoFrame, model) {
    // Preprocess
    var input = preprocessFrame(videoFrame, 300, 300);

    // Inference
    return model.run(input).then(function(output) {
        // Postprocess
        var detections = postprocessDetections(output, 0.5, 0.4);

        // Filter for specific classes (e.g., people only)
        var people = detections.filter(function(det) {
            return det.label === "person";
        });

        return {
            totalDetections: detections.length,
            peopleCount: people.length,
            detections: detections
        };
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

// Conceptual face detection and demographic analysis
function analyzeFaces(videoFrame, faceDetector, demographicModel) {
    // Stage 1: Detect faces
    return faceDetector.run(videoFrame).then(function(faces) {
        if (faces.length === 0) {
            return {faceCount: 0, demographics: []};
        }

        // Stage 2: Analyze each face
        var analyses = faces.map(function(face) {
            var faceCrop = cropRegion(videoFrame, face.box);
            return demographicModel.run(faceCrop);
        });

        return Promise.all(analyses).then(function(demographics) {
            return {
                faceCount: faces.length,
                demographics: demographics  // age, gender estimates
            };
        });
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// Conceptual keyword spotting
function detectKeywords(audioBuffer, model, keywords) {
    // Preprocess audio
    var features = preprocessAudio(audioBuffer, 16000);

    // Run inference
    return model.run(features).then(function(output) {
        var probabilities = softmax(output.logits);

        // Find highest probability keyword
        var maxIdx = argmax(probabilities);
        var confidence = probabilities[maxIdx];

        if (confidence > 0.8) {
            return {
                keyword: keywords[maxIdx],
                confidence: confidence,
                detected: true
            };
        }

        return {detected: false};
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

// Conceptual audio event detection
function detectAudioEvents(audioBuffer, model, eventTypes) {
    var features = extractAudioFeatures(audioBuffer);

    return model.run(features).then(function(output) {
        var events = [];

        for (var i = 0; i < output.length; i++) {
            if (output[i] > 0.7) {
                events.push({
                    type: eventTypes[i],
                    confidence: output[i]
                });
            }
        }

        return events;
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// Conceptual frame skipping for performance
var InferenceThrottler = function(targetFPS) {
    this.targetFPS = targetFPS;
    this.frameInterval = 1000 / targetFPS;
    this.lastInferenceTime = 0;

    this.shouldProcess = function() {
        var now = Date.now();
        if (now - this.lastInferenceTime >= this.frameInterval) {
            this.lastInferenceTime = now;
            return true;
        }
        return false;
    };
};

// Usage
var throttler = new InferenceThrottler(10);  // 10 FPS

function processVideoStream(videoFrame, model) {
    if (throttler.shouldProcess()) {
        runInference(model, videoFrame);
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

// Non-blocking inference with queue
var InferenceQueue = function() {
    this.processing = false;
    this.latestFrame = null;

    this.submit = function(frame) {
        this.latestFrame = frame;  // Always keep latest
        if (!this.processing) {
            this.process();
        }
    };

    this.process = function() {
        var self = this;
        if (this.latestFrame === null) {
            this.processing = false;
            return;
        }

        this.processing = true;
        var frame = this.latestFrame;
        this.latestFrame = null;

        runInference(model, frame).then(function(results) {
            handleResults(results);
            self.process();  // Process next if available
        });
    };
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

// Conceptual performance tracking
var PerformanceMonitor = function() {
    this.inferenceTimes = [];
    this.maxSamples = 100;

    this.recordInference = function(startTime, endTime) {
        var duration = endTime - startTime;
        this.inferenceTimes.push(duration);

        if (this.inferenceTimes.length > this.maxSamples) {
            this.inferenceTimes.shift();
        }
    };

    this.getStats = function() {
        var avg = this.inferenceTimes.reduce(function(a, b) {
            return a + b;
        }, 0) / this.inferenceTimes.length;

        var max = Math.max.apply(null, this.inferenceTimes);
        var min = Math.min.apply(null, this.inferenceTimes);

        return {
            avgMs: avg,
            maxMs: max,
            minMs: min,
            fps: 1000 / avg
        };
    };
};

// Usage
var perfMonitor = new PerformanceMonitor();

function runInferenceWithTracking(model, input) {
    var startTime = Date.now();

    return model.run(input).then(function(results) {
        var endTime = Date.now();
        perfMonitor.recordInference(startTime, endTime);

        var stats = perfMonitor.getStats();
        console.log("Inference: " + stats.avgMs + "ms, " + stats.fps + " FPS");

        return results;
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

var VideoInputClass = require("@brightsign/videoinput");
var vi = new VideoInputClass();

// Get input status
vi.getStatus().then(function(status) {
    if (status.devicePresent) {
        console.log("Camera connected:");
        console.log("Resolution: " + status.width + "x" + status.height);
        console.log("Frame rate: " + status.frameRate);
        console.log("Color space: " + status.colorSpace);

        // Ready for AI processing
        startInferenceOnVideoInput(vi);
    } else {
        console.log("No camera connected");
    }
});

// Monitor for camera connection changes
vi.addEventListener('hdmiinputchange', function(event) {
    console.log("Camera connection changed");
    checkCameraStatus();
});

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// Conceptual sensor fusion
var GpioClass = require("@brightsign/gpio");
var gpio = new GpioClass();

// Motion sensor on GPIO
gpio.enableInput(0);  // PIR motion sensor

var motionDetected = false;

gpio.addEventListener('inputchange', function(event) {
    if (event.input === 0 && event.state === true) {
        motionDetected = true;
        console.log("Motion detected, starting AI vision");
        startVisionProcessing();
    }
});

// Only run expensive vision AI when motion detected
function startVisionProcessing() {
    if (motionDetected) {
        // Run object detection or face recognition
        processVideoFrame(currentFrame);
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

// Conceptual multi-modal AI application
function multiModalAnalysis() {
    var results = {
        visual: null,
        audio: null,
        motion: false,
        timestamp: Date.now()
    };

    // Visual AI
    if (cameraActive) {
        results.visual = detectObjects(currentVideoFrame);
    }

    // Audio AI
    if (audioAvailable) {
        results.audio = classifyAudio(currentAudioBuffer);
    }

    // Motion sensor
    results.motion = gpio.isInputActive(0);

    // Fuse results for decision making
    return fuseResults(results);
}

function fuseResults(results) {
    // Example: High confidence person + voice detected + motion
    if (results.visual && results.visual.peopleCount > 0 &&
        results.audio && results.audio.containsSpeech &&
        results.motion) {
        return {
            event: "person_interacting",
            confidence: 0.95
        };
    }

    return {event: "none", confidence: 0.0};
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

var ControlPortClass = require("@brightsign/controlport");
var controlPort = new ControlPortClass();

// Configure GPIO outputs
controlPort.enableOutput(0);  // LED indicator
controlPort.enableOutput(1);  // Relay for external device

// Control outputs based on AI results
function handleDetectionResults(detections) {
    var personDetected = detections.some(function(det) {
        return det.label === "person" && det.score > 0.7;
    });

    if (personDetected) {
        // Turn on LED
        controlPort.setOutputState(0, true);

        // Trigger external device
        controlPort.setOutputState(1, true);

        // Auto-off after 5 seconds
        setTimeout(function() {
            controlPort.setOutputState(0, false);
            controlPort.setOutputState(1, false);
        }, 5000);
    }
}

// Integrate with inference pipeline
function processFrameWithActions(videoFrame, model) {
    detectObjects(videoFrame, model).then(function(results) {
        handleDetectionResults(results.detections);
        updateDisplay(results);
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Using TensorFlow Lite converter
python -c "
import tensorflow as tf

# Load SavedModel
converter = tf.lite.TFLiteConverter.from_saved_model('saved_model/')

# Optimize
converter.optimizations = [tf.lite.Optimize.DEFAULT]

# Convert
tflite_model = converter.convert()

# Save
with open('model.tflite', 'wb') as f:
    f.write(tflite_model)
"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# Export PyTorch model to ONNX
python -c "
import torch
import torchvision

# Load model
model = torchvision.models.mobilenet_v2(pretrained=True)
model.eval()

# Create dummy input
dummy_input = torch.randn(1, 3, 224, 224)

# Export
torch.onnx.export(
    model,
    dummy_input,
    'mobilenet_v2.onnx',
    input_names=['input'],
    output_names=['output'],
    dynamic_axes={'input': {0: 'batch'}, 'output': {0: 'batch'}}
)
"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Install onnx-tf
pip install onnx-tf

# Convert ONNX to TensorFlow
onnx-tf convert -i model.onnx -o model_tf/

# Convert TensorFlow to TFLite
python -c "
import tensorflow as tf

converter = tf.lite.TFLiteConverter.from_saved_model('model_tf/')
tflite_model = converter.convert()

with open('model.tflite', 'wb') as f:
    f.write(tflite_model)
"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

import numpy as np
import tensorflow as tf

# Load TFLite model
interpreter = tf.lite.Interpreter(model_path='model.tflite')
interpreter.allocate_tensors()

# Get input/output details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

# Test with sample data
def test_model(test_images, test_labels):
    correct = 0
    total = len(test_images)

    for image, label in zip(test_images, test_labels):
        # Preprocess
        input_data = preprocess(image)

        # Run inference
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()
        output = interpreter.get_tensor(output_details[0]['index'])

        # Check prediction
        predicted = np.argmax(output)
        if predicted == label:
            correct += 1

    accuracy = correct / total
    print(f"Accuracy: {accuracy * 100:.2f}%")
    return accuracy

# Run tests
accuracy = test_model(test_images, test_labels)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

import time

def benchmark_model(interpreter, num_runs=100):
    # Warm up
    for _ in range(10):
        interpreter.invoke()

    # Benchmark
    times = []
    for _ in range(num_runs):
        start = time.time()
        interpreter.invoke()
        end = time.time()
        times.append((end - start) * 1000)  # Convert to ms

    avg_time = np.mean(times)
    std_time = np.std(times)
    fps = 1000 / avg_time

    print(f"Avg inference time: {avg_time:.2f}ms ± {std_time:.2f}ms")
    print(f"Throughput: {fps:.1f} FPS")

    return {
        'avg_ms': avg_time,
        'std_ms': std_time,
        'fps': fps
    }

benchmark_model(interpreter)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Enable profiling
interpreter = tf.lite.Interpreter(
    model_path='model.tflite',
    experimental_enable_op_profiling=True
)
interpreter.allocate_tensors()

# Run inference
interpreter.invoke()

# Get profiling information
profiling_data = interpreter.get_tensor_details()

# Analyze per-operator timing
for op in profiling_data:
    print(f"Op: {op['name']}")
    print(f"  Time: {op['execution_time_us']} us")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

def profile_and_analyze(interpreter):
    # Run with profiling
    interpreter.invoke()

    # Get operator details
    ops = []
    for i in range(interpreter._get_ops_details()):
        op_details = interpreter._get_op_details(i)
        ops.append({
            'name': op_details['op_name'],
            'time_us': op_details['execution_time_us']
        })

    # Sort by execution time
    ops.sort(key=lambda x: x['time_us'], reverse=True)

    # Display top slow operations
    print("Top 10 slowest operations:")
    for i, op in enumerate(ops[:10]):
        print(f"{i+1}. {op['name']}: {op['time_us']} us")

    return ops

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

// Conceptual smart signage implementation
function smartSignageController(aiResults) {
    var demographics = aiResults.demographics;
    var peopleCount = aiResults.faceCount;

    // No audience - show default content
    if (peopleCount === 0) {
        displayContent("default.mp4");
        return;
    }

    // Analyze audience
    var avgAge = averageAge(demographics);
    var genderRatio = calculateGenderRatio(demographics);

    // Select content based on audience
    var content;
    if (avgAge < 25 && genderRatio.male > 0.6) {
        content = "young_male_targeted.mp4";
    } else if (avgAge > 50) {
        content = "senior_targeted.mp4";
    } else if (genderRatio.female > 0.7) {
        content = "female_targeted.mp4";
    } else {
        content = "general_audience.mp4";
    }

    displayContent(content);

    // Log analytics
    logAudienceEvent({
        timestamp: Date.now(),
        count: peopleCount,
        avgAge: avgAge,
        genderRatio: genderRatio,
        contentShown: content
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

// Process demographics without storing faces
function privacyCompliantAnalytics(videoFrame, faceModel) {
    return faceModel.run(videoFrame).then(function(results) {
        // Extract only aggregate demographics
        var analytics = {
            timestamp: Date.now(),
            count: results.faceCount,
            ageGroups: {
                under18: 0,
                age18to35: 0,
                age36to55: 0,
                over55: 0
            },
            gender: {
                male: 0,
                female: 0
            },
            attentionTime: results.avgAttentionTime
        };

        // Aggregate demographics (no individual data stored)
        results.demographics.forEach(function(demo) {
            // Age grouping
            if (demo.estimatedAge < 18) analytics.ageGroups.under18++;
            else if (demo.estimatedAge < 36) analytics.ageGroups.age18to35++;
            else if (demo.estimatedAge < 56) analytics.ageGroups.age36to55++;
            else analytics.ageGroups.over55++;

            // Gender
            if (demo.estimatedGender === 'male') analytics.gender.male++;
            else analytics.gender.female++;
        });

        // Store only aggregated data
        return analytics;
    });
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

// Activate display when person approaches
function presenceBasedActivation(objectDetectionResults) {
    var people = objectDetectionResults.detections.filter(function(d) {
        return d.label === "person";
    });

    if (people.length === 0) {
        // No one present - enter sleep mode
        if (displayActive) {
            setTimeout(function() {
                setDisplayPower(false);
                displayActive = false;
            }, 30000);  // 30 second timeout
        }
    } else {
        // Person detected - wake display
        if (!displayActive) {
            setDisplayPower(true);
            displayActive = true;
            showWelcomeScreen();
        }

        // Estimate distance from bounding box size
        var distance = estimateDistance(people[0].box);

        if (distance < 1.5) {  // Within 1.5 meters
            showInteractiveContent();
        } else {
            showAttractLoop();
        }
    }
}

function estimateDistance(boundingBox) {
    // Larger box = closer person
    var boxHeight = boundingBox.bottom - boundingBox.top;
    var normalizedHeight = boxHeight / imageHeight;

    // Empirical formula (calibrate for your setup)
    var distance = 2.0 / normalizedHeight;
    return distance;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

// Hand gesture detection for touchless control
function gestureControl(poseResults) {
    var hands = poseResults.hands;

    if (hands.length === 0) return;

    var rightHand = hands.find(function(h) { return h.handedness === 'right'; });
    if (!rightHand) return;

    // Detect gestures
    var gesture = classifyGesture(rightHand.keypoints);

    switch(gesture) {
        case 'wave':
            showGreeting();
            break;
        case 'swipe_left':
            navigateContent('previous');
            break;
        case 'swipe_right':
            navigateContent('next');
            break;
        case 'point':
            var target = getPointingTarget(rightHand);
            selectItem(target);
            break;
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

// Track viewer attention and dwell time
var AudienceAnalytics = function() {
    this.sessions = [];
    this.currentSession = null;

    this.update = function(detectionResults) {
        var peopleCount = detectionResults.peopleCount;
        var timestamp = Date.now();

        if (peopleCount > 0) {
            if (this.currentSession === null) {
                // Start new session
                this.currentSession = {
                    startTime: timestamp,
                    endTime: null,
                    peakCount: peopleCount,
                    avgCount: peopleCount,
                    samples: 1
                };
            } else {
                // Update existing session
                this.currentSession.samples++;
                this.currentSession.avgCount =
                    (this.currentSession.avgCount * (this.currentSession.samples - 1) +
                     peopleCount) / this.currentSession.samples;
                this.currentSession.peakCount =
                    Math.max(this.currentSession.peakCount, peopleCount);
            }
        } else {
            if (this.currentSession !== null) {
// ... (see full example below)
    detectObjects(currentFrame, model).then(function(results) {
        analytics.update(results);
    });
}, 1000);  // Update every second

// Generate daily report
setInterval(function() {
    var report = analytics.getReport();
    sendAnalyticsReport(report);
}, 86400000);  // Every 24 hours

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

// Content selection based on real-time conditions
function automaticContentSelection(aiInputs, timeOfDay, weather) {
    var score = {};

    // Score each content item
    contentLibrary.forEach(function(item) {
        score[item.id] = 0;

        // Audience demographics
        if (aiInputs.demographics) {
            if (matchesDemographic(item.targetDemo, aiInputs.demographics)) {
                score[item.id] += 30;
            }
        }

        // Time of day
        if (item.scheduleWeights) {
            var hour = new Date().getHours();
            score[item.id] += item.scheduleWeights[hour] || 0;
        }

        // Weather conditions
        if (weather && item.weatherTags) {
            if (item.weatherTags.includes(weather.condition)) {
                score[item.id] += 20;
            }
        }

        // Crowd size
        if (aiInputs.peopleCount > 5 && item.tags.includes('group')) {
// ... (see full example below)
            );

            // Update playlist if content changed
            if (content.id !== currentContent.id) {
                transitionToContent(content);
                logContentChange(content, aiResults);
            }
        });
    }, 10000);  // Re-evaluate every 10 seconds
}