diff --git a/api/src/controllers/recognize.controller.js b/api/src/controllers/recognize.controller.js index 3b5175ba..c44fa85d 100644 --- a/api/src/controllers/recognize.controller.js +++ b/api/src/controllers/recognize.controller.js @@ -148,7 +148,7 @@ module.exports.start = async (req, res) => { ); } - const { best, misses, unknown, results, attempts } = recognize.normalize( + const { best, misses, unknowns, results, attempts, counts } = recognize.normalize( await Promise.all(promises) ); @@ -160,10 +160,11 @@ module.exports.start = async (req, res) => { attempts, camera, zones, + counts, matches: best, misses, + unknowns, }; - if (unknown && Object.keys(unknown).length) output.unknown = unknown; if (AUTH) output.token = jwt.sign({ route: 'storage', expiresIn: TOKEN.IMAGE }); if (resultsOutput === 'all') output.results = results; @@ -171,17 +172,16 @@ module.exports.start = async (req, res) => { console.log(`done processing ${camera}: ${id} in ${duration} sec`); const loggedOutput = JSON.parse(JSON.stringify(output)); - ['matches', 'misses'].forEach((type) => + ['matches', 'misses', 'unknowns'].forEach((type) => loggedOutput[type].forEach((result) => delete result.base64) ); - if (loggedOutput.unknown) delete loggedOutput.unknown.base64; console.log(loggedOutput); PROCESSING = false; res.send(output); - recognize.save.latest(camera, best, misses, unknown); + recognize.save.latest(camera, best, misses, unknowns[0]); mqtt.recognize(output); notify.publish(output, camera, results); if (event.type === 'frigate') frigate.subLabel(event.topic, id, best); diff --git a/api/src/util/mqtt.util.js b/api/src/util/mqtt.util.js index 96ace864..5c5c68e1 100644 --- a/api/src/util/mqtt.util.js +++ b/api/src/util/mqtt.util.js @@ -3,7 +3,6 @@ const { v4: uuidv4 } = require('uuid'); const axios = require('axios'); const mqtt = require('mqtt'); const fs = require('./fs.util'); -const { contains } = require('./helpers.util'); const { jwt } = require('./auth.util'); const { AUTH, SERVER, MQTT, FRIGATE, CAMERAS, STORAGE, UI } = require('../constants')(); const config = require('../constants/config'); @@ -156,47 +155,51 @@ module.exports.subscribe = () => { module.exports.recognize = (data) => { try { if (!MQTT || !MQTT.HOST) return; - const { matches, misses, unknown } = data; - const camera = data.camera.toLowerCase(); - const hasUnknown = unknown && Object.keys(unknown).length; - - const configData = JSON.parse(JSON.stringify(data)); - delete configData.matches; - delete configData.unknown; - delete configData.results; + const baseData = JSON.parse(JSON.stringify(data)); + const { id, duration, timestamp, attempts, zones, matches, misses, unknowns, counts } = + baseData; + const camera = baseData.camera.toLowerCase(); + + const payload = { + base: { + id, + duration, + timestamp, + attempts, + camera, + zones, + }, + }; + payload.unknown = { ...payload.base, unknown: unknowns[0], unknowns }; + payload.match = { ...payload.base }; + payload.camera = { + ...payload.base, + matches, + misses, + unknowns, + counts, + }; + payload.cameraReset = { + ...payload.camera, + counts: { + person: 0, + match: 0, + miss: 0, + unknown: 0, + }, + }; const messages = []; - const persons = [...new Set([...matches, ...misses].map(({ name }) => name))]; - let personCount = persons.length ? persons.length : hasUnknown ? 1 : 0; - // check to see if unknown bounding box is contained within or contains any of the match bounding boxes - // if false, then add 1 to the person count - if (persons.length && hasUnknown) { - let unknownFoundInMatch = false; - matches.forEach((match) => { - if (contains(match.box, unknown.box) || contains(unknown.box, match.box)) - unknownFoundInMatch = true; - }); - - let unknownFoundInMiss = false; - misses.forEach((miss) => { - if (contains(miss.box, unknown.box) || contains(unknown.box, miss.box)) - unknownFoundInMiss = true; - }); - if (!unknownFoundInMatch && !unknownFoundInMiss) personCount += 1; - } messages.push({ topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`, - message: personCount.toString(), + message: counts.person.toString(), }); - if (hasUnknown) { + if (unknowns.length) { messages.push({ topic: `${MQTT.TOPICS.MATCHES}/unknown`, - message: JSON.stringify({ - ...configData, - unknown, - }), + message: JSON.stringify(payload.unknown), }); if (MQTT.TOPICS.HOMEASSISTANT) { @@ -215,10 +218,7 @@ module.exports.recognize = (data) => { messages.push({ topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/unknown/state`, - message: JSON.stringify({ - ...configData, - unknown, - }), + message: JSON.stringify(payload.unknown), }); } } @@ -230,7 +230,7 @@ module.exports.recognize = (data) => { messages.push({ topic: `${MQTT.TOPICS.MATCHES}/${topic}`, message: JSON.stringify({ - ...configData, + ...payload.match, match, }), }); @@ -252,22 +252,17 @@ module.exports.recognize = (data) => { messages.push({ topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${topic}/state`, message: JSON.stringify({ - ...configData, + ...payload.match, match, }), }); } }); - if (matches.length || misses.length || hasUnknown) { + if (matches.length || misses.length || unknowns.length) { messages.push({ topic: `${MQTT.TOPICS.CAMERAS}/${camera}`, - message: JSON.stringify({ - ...configData, - matches, - misses, - unknown, - }), + message: JSON.stringify(payload.camera), }); if (MQTT.TOPICS.HOMEASSISTANT) { @@ -286,13 +281,7 @@ module.exports.recognize = (data) => { messages.push({ topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`, - message: JSON.stringify({ - ...configData, - matches, - misses, - unknown, - personCount, - }), + message: JSON.stringify(payload.camera), }); } } @@ -305,12 +294,7 @@ module.exports.recognize = (data) => { if (MQTT.TOPICS.HOMEASSISTANT) { this.publish({ topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/double-take/${camera}/state`, - message: JSON.stringify({ - ...configData, - matches, - unknown, - personCount: 0, - }), + message: JSON.stringify(payload.cameraReset), }); } }, 30000); diff --git a/api/src/util/recognize.util.js b/api/src/util/recognize.util.js index 22c413d9..3a554894 100644 --- a/api/src/util/recognize.util.js +++ b/api/src/util/recognize.util.js @@ -39,28 +39,33 @@ module.exports.save = { module.exports.normalize = (results = []) => { const best = { matches: [], misses: [] }; - const tmp = { matches: {}, misses: {} }; - let unknown = {}; + const tmp = { matches: {}, misses: {}, counts: {} }; + const unknowns = []; let attempts = 0; results.forEach((group) => { attempts += group.attempts; group.results.forEach((attempt) => { + const face = tmp.counts[attempt.detector]; + if (face) + tmp.counts[attempt.detector] = { + count: face.count + attempt.results.length, + attempts: (face.attempts += 1), + }; + else tmp.counts[attempt.detector] = { count: attempt.results.length, attempts: 1 }; + const matches = attempt.results.filter((obj) => obj.match); const misses = attempt.results.filter((obj) => !obj.match && obj.name !== 'unknown'); - const unknowns = attempt.results.filter((obj) => obj.name === 'unknown'); - - unknowns.forEach((obj) => { - if (unknown.confidence === undefined || unknown.confidence < obj.confidence) { - unknown = { - ...obj, - type: group.type, - duration: attempt.duration, - detector: attempt.detector, - filename: attempt.filename, - base64: attempt.base64 || null, - }; - } + const tmpUnknowns = attempt.results.filter((obj) => obj.name === 'unknown'); + tmpUnknowns.forEach((obj) => { + unknowns.push({ + ...obj, + type: group.type, + duration: attempt.duration, + detector: attempt.detector, + filename: attempt.filename, + base64: attempt.base64 || null, + }); }); matches.forEach((match) => { @@ -100,5 +105,23 @@ module.exports.normalize = (results = []) => { for (const value of Object.values(tmp.matches)) best.matches.push(value); for (const value of Object.values(tmp.misses)) best.misses.push(value); - return { best: best.matches, misses: best.misses, results, attempts, unknown }; + let personCount = 0; + for (const [, value] of Object.entries(tmp.counts)) + personCount += Math.round(value.count / value.attempts); + + const counts = { + person: personCount ? Math.round(personCount / Object.keys(tmp.counts).length) : 0, + match: best.matches.length, + miss: best.misses.length, + }; + counts.unknown = counts.person - counts.match - counts.miss; + + return { + best: best.matches, + misses: best.misses, + results, + attempts, + unknowns: unknowns.sort((a, b) => b.confidence - a.confidence), + counts, + }; };