From 16b22f213da392ab5768ab6e50e6d8a7b6fc2177 Mon Sep 17 00:00:00 2001 From: gyuhyuk Date: Tue, 21 Nov 2023 01:42:46 +0900 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20game-room=20webcam=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .eslintrc | 65 ++++---- package.json | 7 + src/App.tsx | 3 + src/assets/pages/Game/GameRoom.js | 74 +++++++++ src/assets/pages/Game/params.tsx | 124 ++++++++++++++ src/assets/pages/Game/util.js | 261 ++++++++++++++++++++++++++++++ yarn.lock | 190 +++++++++++++++++++++- 7 files changed, 688 insertions(+), 36 deletions(-) create mode 100644 src/assets/pages/Game/GameRoom.js create mode 100644 src/assets/pages/Game/params.tsx create mode 100644 src/assets/pages/Game/util.js diff --git a/.eslintrc b/.eslintrc index 25bb108..cb9ed26 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,34 +1,35 @@ { - "env": { - "browser": true, - "es2021": true - }, - "extends": [ - "plugin:prettier/recommended", - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:react/recommended" + "env": { + "browser": true, + "es2021": true + }, + "extends": [ + "plugin:prettier/recommended", + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:react/recommended" + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint", "react"], + "rules": { + "no-unused-vars": "off", + "@typescript-eslint/no-unused-vars": ["warn"], + "no-use-before-define": "off", + "no-empty-function": "warn", + "react/prop-types": "off", + "react/jsx-one-expression-per-line": "off", + "react/jsx-props-no-spreading": "off", + "react/react-in-jsx-scope": "off", + "prettier/prettier": ["error", { "endOfLine": "auto" }], + "@typescript-eslint/no-empty-interface": [ + "warn", + { "allowSingleExtends": false } ], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "plugins": ["@typescript-eslint", "react"], - "rules": { - "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": ["warn"], - "no-use-before-define": "off", - "no-empty-function": "warn", - "react/prop-types": "off", - "react/jsx-one-expression-per-line": "off", - "react/jsx-props-no-spreading": "off", - "react/react-in-jsx-scope": "off", - "prettier/prettier": ["error", { "endOfLine": "auto" }], - "@typescript-eslint/no-empty-interface": [ - "warn", - { "allowSingleExtends": false } - ], - "@typescript-eslint/no-empty-function": "warn" - } - } \ No newline at end of file + "@typescript-eslint/no-empty-function": "warn", + "@typescript-eslint/no-explicit-any": ["off"] + } +} diff --git a/package.json b/package.json index 5e0bade..580c6c6 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,11 @@ "private": true, "dependencies": { "@hookform/resolvers": "^3.3.2", + "@mediapipe/pose": "^0.5.1675469404", + "@tensorflow-models/pose-detection": "^2.1.3", + "@tensorflow/tfjs": "^4.13.0", + "@tensorflow/tfjs-backend-webgl": "^4.13.0", + "@tensorflow/tfjs-backend-webgpu": "^4.13.0", "@testing-library/jest-dom": "^5.14.1", "@testing-library/react": "^13.0.0", "@testing-library/user-event": "^13.2.1", @@ -17,7 +22,9 @@ "react-hook-form": "^7.47.0", "react-router-dom": "^6.16.0", "react-scripts": "5.0.1", + "react-webcam": "^7.2.0", "recoil": "^0.7.7", + "scatter-gl": "^0.0.13", "styled-components": "^6.0.9", "styled-reset": "^4.5.1", "typescript": "^5.2.2", diff --git a/src/App.tsx b/src/App.tsx index a452b36..9f88480 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,6 +9,8 @@ import SquatGameGuide from './assets/modal/SquatGameGuide'; import Dashboard from './assets/pages/Dashboard'; import MainHeader from './assets/component/MainHeader'; +import GameRoom from './assets/pages/Game/GameRoom'; + function App() { return ( @@ -17,6 +19,7 @@ function App() { }> } /> + } /> diff --git a/src/assets/pages/Game/GameRoom.js b/src/assets/pages/Game/GameRoom.js new file mode 100644 index 0000000..4d6ec35 --- /dev/null +++ b/src/assets/pages/Game/GameRoom.js @@ -0,0 +1,74 @@ +import { useRef, useEffect } from 'react'; +import '@tensorflow/tfjs-backend-webgl'; +import * as poseDetection from '@tensorflow-models/pose-detection'; +import Webcam from 'react-webcam'; +import * as tf from '@tensorflow/tfjs-core'; + +const GameRoom = () => { + const webcamRef = useRef(null); + // const canvasRef = useRef(null); + + const runBlazePose = async () => { + const detector = await poseDetection.createDetector( + poseDetection.SupportedModels.BlazePose, + { + runtime: 'tfjs', + }, + ); + + setInterval(() => { + detect(detector); + }, 20); + }; + + useEffect(() => { + const initializeTensorflow = async () => { + await tf.ready(); + runBlazePose(); + }; + + initializeTensorflow(); + }, []); + + const detect = async (net) => { + if ( + typeof webcamRef.current !== 'undefined' && + webcamRef.current !== null && + webcamRef.current.video.readyState === 4 + ) { + const video = webcamRef.current.video; + const videoWidth = video.videoWidth; + const videoHeight = video.videoHeight; + + webcamRef.current.video.width = videoWidth; + webcamRef.current.video.height = videoHeight; + + const pose = await net.estimatePoses(video); + console.log(pose); + // drawCanvas(pose, videoWidth, videoHeight, canvasRef); + } + }; + + window.requestAnimationFrame(runBlazePose); + return ( +
+ +
+ ); +}; + +export default GameRoom; diff --git a/src/assets/pages/Game/params.tsx b/src/assets/pages/Game/params.tsx new file mode 100644 index 0000000..58d9deb --- /dev/null +++ b/src/assets/pages/Game/params.tsx @@ -0,0 +1,124 @@ +/** + * @license + * Copyright 2021 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as posedetection from '@tensorflow-models/pose-detection'; +// import { isiOS } from './util'; + +export const DEFAULT_LINE_WIDTH = 2; +export const DEFAULT_RADIUS = 4; + +export const VIDEO_SIZE = { + '640 X 480': { width: 640, height: 480 }, + '640 X 360': { width: 640, height: 360 }, + '360 X 270': { width: 360, height: 270 }, +}; +export const STATE = { + camera: { targetFPS: 60, sizeOption: '640 X 480' }, + backend: '', + flags: {}, + modelConfig: {}, +}; + +export const BLAZEPOSE_CONFIG = { + maxPoses: 1, + type: 'full', + scoreThreshold: 0.65, + render3D: true, +}; + +// export const STATE_FOR_BLAZEMODEL = { +// camera: { targetFPS: 60, sizeOption: "640 X 480" }, +// backend: "mediapipe-gpu", +// flags: {}, +// modelConfig: { ...BLAZEPOSE_CONFIG }, +// model: posedetection.SupportedModels.BlazePose, +// }; + +// export const POSENET_CONFIG = { +// maxPoses: 1, +// scoreThreshold: 0.5 +// }; +export const MOVENET_CONFIG = { + maxPoses: 1, + type: 'lightning', + scoreThreshold: 0.3, + customModel: '', + enableTracking: false, +}; +/** + * This map descripes tunable flags and theior corresponding types. + * + * The flags (keys) in the map satisfy the following two conditions: + * - Is tunable. For example, `IS_BROWSER` and `IS_CHROME` is not tunable, + * because they are fixed when running the scripts. + * - Does not depend on other flags when registering in `ENV.registerFlag()`. + * This rule aims to make the list streamlined, and, since there are + * dependencies between flags, only modifying an independent flag without + * modifying its dependents may cause inconsistency. + * (`WEBGL_RENDER_FLOAT32_CAPABLE` is an exception, because only exposing + * `WEBGL_FORCE_F16_TEXTURES` may confuse users.) + */ +export const TUNABLE_FLAG_VALUE_RANGE_MAP = { + WEBGL_VERSION: [1, 2], + WASM_HAS_SIMD_SUPPORT: [true, false], + WASM_HAS_MULTITHREAD_SUPPORT: [true, false], + WEBGL_CPU_FORWARD: [true, false], + WEBGL_PACK: [true, false], + WEBGL_FORCE_F16_TEXTURES: [true, false], + WEBGL_RENDER_FLOAT32_CAPABLE: [true, false], + WEBGL_FLUSH_THRESHOLD: [-1, 0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], + CHECK_COMPUTATION_FOR_ERRORS: [true, false], +}; + +export const BACKEND_FLAGS_MAP = { + ['tfjs-wasm']: ['WASM_HAS_SIMD_SUPPORT', 'WASM_HAS_MULTITHREAD_SUPPORT'], + ['tfjs-webgl']: [ + 'WEBGL_VERSION', + 'WEBGL_CPU_FORWARD', + 'WEBGL_PACK', + 'WEBGL_FORCE_F16_TEXTURES', + 'WEBGL_RENDER_FLOAT32_CAPABLE', + 'WEBGL_FLUSH_THRESHOLD', + ], + ['tfjs-webgpu']: [], + ['mediapipe-gpu']: [], +}; + +export const MODEL_BACKEND_MAP = { + // [posedetection.SupportedModels.PoseNet]: ['tfjs-webgl', 'tfjs-webgpu'], + [posedetection.SupportedModels.MoveNet]: [ + 'tfjs-webgl', + 'tfjs-wasm', + 'tfjs-webgpu', + ], + [posedetection.SupportedModels.BlazePose]: [ + 'mediapipe-gpu', + 'tfjs-webgl', + 'tfjs-webgpu', + ], +}; + +export const TUNABLE_FLAG_NAME_MAP = { + PROD: 'production mode', + WEBGL_VERSION: 'webgl version', + WASM_HAS_SIMD_SUPPORT: 'wasm SIMD', + WASM_HAS_MULTITHREAD_SUPPORT: 'wasm multithread', + WEBGL_CPU_FORWARD: 'cpu forward', + WEBGL_PACK: 'webgl pack', + WEBGL_FORCE_F16_TEXTURES: 'enforce float16', + WEBGL_RENDER_FLOAT32_CAPABLE: 'enable float32', + WEBGL_FLUSH_THRESHOLD: 'GL flush wait time(ms)', +}; diff --git a/src/assets/pages/Game/util.js b/src/assets/pages/Game/util.js new file mode 100644 index 0000000..9282489 --- /dev/null +++ b/src/assets/pages/Game/util.js @@ -0,0 +1,261 @@ +/** + * @license + * Copyright 2023 Google LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import * as posedetection from '@tensorflow-models/pose-detection'; +import * as scatter from 'scatter-gl'; + +import * as params from './params'; + +// These anchor points allow the pose pointcloud to resize according to its +// position in the input. +const ANCHOR_POINTS = [ + [0, 0, 0], + [0, 1, 0], + [-1, 0, 0], + [-1, -1, 0], +]; + +// #ffffff - White +// #800000 - Maroon +// #469990 - Malachite +// #e6194b - Crimson +// #42d4f4 - Picton Blue +// #fabed4 - Cupid +// #aaffc3 - Mint Green +// #9a6324 - Kumera +// #000075 - Navy Blue +// #f58231 - Jaffa +// #4363d8 - Royal Blue +// #ffd8b1 - Caramel +// #dcbeff - Mauve +// #808000 - Olive +// #ffe119 - Candlelight +// #911eb4 - Seance +// #bfef45 - Inchworm +// #f032e6 - Razzle Dazzle Rose +// #3cb44b - Chateau Green +// #a9a9a9 - Silver Chalice +const COLOR_PALETTE = [ + '#ffffff', + '#800000', + '#469990', + '#e6194b', + '#42d4f4', + '#fabed4', + '#aaffc3', + '#9a6324', + '#000075', + '#f58231', + '#4363d8', + '#ffd8b1', + '#dcbeff', + '#808000', + '#ffe119', + '#911eb4', + '#bfef45', + '#f032e6', + '#3cb44b', + '#a9a9a9', +]; +export class RendererCanvas2d { + constructor(canvas) { + this.ctx = canvas.getContext('2d'); + this.scatterGLEl = document.querySelector('#scatter-gl-container'); + this.scatterGL = new scatter.ScatterGL(this.scatterGLEl, { + rotateOnStart: true, + selectEnabled: false, + styles: { polyline: { defaultOpacity: 1, deselectedOpacity: 1 } }, + }); + this.scatterGLHasInitialized = false; + this.videoWidth = canvas.width; + this.videoHeight = canvas.height; + this.flip(this.videoWidth, this.videoHeight); + } + + flip(videoWidth, videoHeight) { + // Because the image from camera is mirrored, need to flip horizontally. + this.ctx.translate(videoWidth, 0); + this.ctx.scale(-1, 1); + + this.scatterGLEl.style = `width: ${videoWidth}px; height: ${videoHeight}px;`; + this.scatterGL.resize(); + + this.scatterGLEl.style.display = params.STATE.modelConfig.render3D + ? 'inline-block' + : 'none'; + } + + draw(rendererParams) { + const [video, poses, isModelChanged] = rendererParams; + this.drawCtx(video); + + // The null check makes sure the UI is not in the middle of changing to a + // different model. If during model change, the result is from an old model, + // which shouldn't be rendered. + if (poses && poses.length > 0 && !isModelChanged) { + this.drawResults(poses); + } + } + + drawCtx(video) { + this.ctx.drawImage(video, 0, 0, this.videoWidth, this.videoHeight); + } + + clearCtx() { + this.ctx.clearRect(0, 0, this.videoWidth, this.videoHeight); + } + + /** + * Draw the keypoints and skeleton on the video. + * @param poses A list of poses to render. + */ + drawResults(poses) { + for (const pose of poses) { + this.drawResult(pose); + } + } + + /** + * Draw the keypoints and skeleton on the video. + * @param pose A pose with keypoints to render. + */ + drawResult(pose) { + if (pose.keypoints != null) { + // this.drawKeypoints(pose.keypoints); + // this.drawSkeleton(pose.keypoints, pose.id); + } + if (pose.keypoints3D != null && params.STATE.modelConfig.render3D) { + this.drawKeypoints3D(pose.keypoints3D); + } + } + + /** + * Draw the keypoints on the video. + * @param keypoints A list of keypoints. + */ + drawKeypoints(keypoints) { + const keypointInd = posedetection.util.getKeypointIndexBySide( + params.STATE.model, + ); + this.ctx.fillStyle = 'Red'; + this.ctx.strokeStyle = 'White'; + this.ctx.lineWidth = params.DEFAULT_LINE_WIDTH; + + for (const i of keypointInd.middle) { + this.drawKeypoint(keypoints[i]); + } + + this.ctx.fillStyle = 'Green'; + for (const i of keypointInd.left) { + this.drawKeypoint(keypoints[i]); + } + + this.ctx.fillStyle = 'Orange'; + for (const i of keypointInd.right) { + this.drawKeypoint(keypoints[i]); + } + } + + drawKeypoint(keypoint) { + // If score is null, just show the keypoint. + const score = keypoint.score != null ? keypoint.score : 1; + const scoreThreshold = params.STATE.modelConfig.scoreThreshold || 0; + + if (score >= scoreThreshold) { + const circle = new Path2D(); + circle.arc(keypoint.x, keypoint.y, params.DEFAULT_RADIUS, 0, 2 * Math.PI); + this?.ctx.fill(circle); + this?.ctx.stroke(circle); + } + } + + /** + * Draw the skeleton of a body on the video. + * @param keypoints A list of keypoints. + */ + drawSkeleton(keypoints, poseId) { + // Each poseId is mapped to a color in the color palette. + const color = + params.STATE.modelConfig.enableTracking && poseId != null + ? COLOR_PALETTE[poseId % 20] + : 'White'; + this.ctx.fillStyle = color; + this.ctx.strokeStyle = color; + this.ctx.lineWidth = params.DEFAULT_LINE_WIDTH; + + posedetection.util + .getAdjacentPairs(params.STATE.model) + .forEach(([i, j]) => { + const kp1 = keypoints[i]; + const kp2 = keypoints[j]; + + // If score is null, just show the keypoint. + const score1 = kp1.score != null ? kp1.score : 1; + const score2 = kp2.score != null ? kp2.score : 1; + const scoreThreshold = params.STATE.modelConfig.scoreThreshold || 0; + + if (score1 >= scoreThreshold && score2 >= scoreThreshold) { + this.ctx.beginPath(); + this.ctx.moveTo(kp1.x, kp1.y); + this.ctx.lineTo(kp2.x, kp2.y); + this.ctx.stroke(); + } + }); + } + + drawKeypoints3D(keypoints) { + const scoreThreshold = params.STATE.modelConfig.scoreThreshold || 0; + const pointsData = keypoints.map((keypoint) => [ + -keypoint.x, + -keypoint.y, + -keypoint.z, + ]); + + const dataset = new scatter.ScatterGL.Dataset([ + ...pointsData, + ...ANCHOR_POINTS, + ]); + + const keypointInd = posedetection.util.getKeypointIndexBySide( + params.STATE.model, + ); + this.scatterGL.setPointColorer((i) => { + if (keypoints[i] == null || keypoints[i].score < scoreThreshold) { + // hide anchor points and low-confident points. + return '#ffffff'; + } + if (i === 0) { + return '#ff0000' /* Red */; + } + if (keypointInd.left.indexOf(i) > -1) { + return '#00ff00' /* Green */; + } + if (keypointInd.right.indexOf(i) > -1) { + return '#ffa500' /* Orange */; + } + }); + + if (!this.scatterGLHasInitialized) { + this.scatterGL.render(dataset); + } else { + this.scatterGL.updateDataset(dataset); + } + const connections = posedetection.util.getAdjacentPairs(params.STATE.model); + const sequences = connections.map((pair) => ({ indices: pair })); + this.scatterGL.setSequences(sequences); + this.scatterGLHasInitialized = true; + } +} diff --git a/yarn.lock b/yarn.lock index c1b1440..2c142fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1667,6 +1667,11 @@ resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A== +"@mediapipe/pose@^0.5.1675469404": + version "0.5.1675469404" + resolved "https://registry.yarnpkg.com/@mediapipe/pose/-/pose-0.5.1675469404.tgz#8f81e64c6561b2357a021a134b54de0204bafc72" + integrity sha512-DFZsNWTsSphRIZppnUCuunzBiHP2FdJXR9ehc7mMi4KG+oPaOH0Em3d6kr7Py+TSyTXC1doH88KcF28k2sBxsQ== + "@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents.3": version "2.1.8-no-fsevents.3" resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz#323d72dd25103d0c4fbdce89dadf574a787b1f9b" @@ -1911,6 +1916,88 @@ "@svgr/plugin-svgo" "^5.5.0" loader-utils "^2.0.0" +"@tensorflow-models/pose-detection@^2.1.3": + version "2.1.3" + resolved "https://registry.yarnpkg.com/@tensorflow-models/pose-detection/-/pose-detection-2.1.3.tgz#5b57f5fe290f7b2c0abbee07607e39a30d3cf3ac" + integrity sha512-SEq4K8fmkVVh9U2VemS2kKuT6DDZf3mO+fcid6F2/DSYMtVrGU13kWo0Q+GAwLznIPaNhlOxVVUgWiDXKUPlzA== + dependencies: + rimraf "^3.0.2" + tslib "2.4.0" + +"@tensorflow/tfjs-backend-cpu@4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.13.0.tgz#086e737bbdadd66f7b6ad8971f3fa7cc0c05ec73" + integrity sha512-k44G+2WZShxI2ejvQdsSQcicFMNWaccsf6bkI0R7dol9t9uj73yg7JkiT0U0uuJE6XwXymJgDe+KJVprg3fAgA== + dependencies: + "@types/seedrandom" "^2.4.28" + seedrandom "^3.0.5" + +"@tensorflow/tfjs-backend-webgl@4.13.0", "@tensorflow/tfjs-backend-webgl@^4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.13.0.tgz#cc15b1d96343e1339c4b0ecc19c979573cbf9423" + integrity sha512-UDwn6o70GyZaVxWdGWrWYJad2tUbxqgLtGfZI19j5EmM554PVsGLd+VHOqv4XodTviawuNq/GzqSdqhqsp8f5w== + dependencies: + "@tensorflow/tfjs-backend-cpu" "4.13.0" + "@types/offscreencanvas" "~2019.3.0" + "@types/seedrandom" "^2.4.28" + seedrandom "^3.0.5" + +"@tensorflow/tfjs-backend-webgpu@^4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgpu/-/tfjs-backend-webgpu-4.13.0.tgz#f233fd7b5dfd32cdea64c3245a38879c5ee99707" + integrity sha512-uVvvvx24+5yvZrugNFj4WmEoYUMjmHSQfH6/nXQ/u6npTnO9Imx0oIiinqgIhgPzUEdIE3kjxRbgpHCu8YGbCA== + dependencies: + "@tensorflow/tfjs-backend-cpu" "4.13.0" + +"@tensorflow/tfjs-converter@4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-4.13.0.tgz#f8ede506ec30948653d6bc08e85bd1d6e8b60f82" + integrity sha512-jA2/IigBXReZHS8Bo308HG7oVzsNPnPgSYfXneRXnxUz+WfcIPkJ6zp48KERZSPja8vOO5eNG4lsUkQpbtiyyw== + +"@tensorflow/tfjs-core@4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-4.13.0.tgz#7d0c472c99be14473c2cd46d529c8b7decece283" + integrity sha512-vvz/kHakvv5Tppp2GDTUBA2/XkNmEkManbdsFEXfwVc5+rVMPEMsRFOjsKTy/TpDRd/4wsJBA99L4F7iG2tr/Q== + dependencies: + "@types/long" "^4.0.1" + "@types/offscreencanvas" "~2019.7.0" + "@types/seedrandom" "^2.4.28" + "@webgpu/types" "0.1.30" + long "4.0.0" + node-fetch "~2.6.1" + seedrandom "^3.0.5" + +"@tensorflow/tfjs-data@4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-data/-/tfjs-data-4.13.0.tgz#c1d4f96748132782877ff13687f4715c7e53366e" + integrity sha512-8FmvzGKBH3SJ3Y+vDTF/coFxD/FMh93YRZHxevNGE+nJcs3JK0grRbjSX3AAWb2GXtz2/o30BU0YL8bW8POuUA== + dependencies: + "@types/node-fetch" "^2.1.2" + node-fetch "~2.6.1" + string_decoder "^1.3.0" + +"@tensorflow/tfjs-layers@4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-layers/-/tfjs-layers-4.13.0.tgz#99bba84941945f3534174d5ba4352907d0b1449b" + integrity sha512-YoBqtVTnE71h48+f89G6ZSYZMN+QsUMccopSxQC6XscncB6Gt1KwuWfpDc2Ld5JeubmUzKLqHdEP0jXIWxssJw== + +"@tensorflow/tfjs@^4.13.0": + version "4.13.0" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs/-/tfjs-4.13.0.tgz#771ae0b007e728737b6a8268f37c5a38a6b48ed1" + integrity sha512-yvjcNMt1q9CLUeOVwoNf0KyMg//fY9earGQGH91C+NcacOK4j0BJUJUqMolEJqfHIbmK2n2CIFmdvgA5epVPSA== + dependencies: + "@tensorflow/tfjs-backend-cpu" "4.13.0" + "@tensorflow/tfjs-backend-webgl" "4.13.0" + "@tensorflow/tfjs-converter" "4.13.0" + "@tensorflow/tfjs-core" "4.13.0" + "@tensorflow/tfjs-data" "4.13.0" + "@tensorflow/tfjs-layers" "4.13.0" + argparse "^1.0.10" + chalk "^4.1.0" + core-js "3.29.1" + regenerator-runtime "^0.13.5" + yargs "^16.0.3" + "@testing-library/dom@^8.5.0": version "8.20.1" resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-8.20.1.tgz#2e52a32e46fc88369eef7eef634ac2a192decd9f" @@ -2157,6 +2244,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/long@^4.0.1": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + "@types/mime@*": version "3.0.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.2.tgz#c1ae807f13d308ee7511a5b81c74f327028e66e8" @@ -2167,6 +2259,14 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.3.tgz#bbe64987e0eb05de150c305005055c7ad784a9ce" integrity sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg== +"@types/node-fetch@^2.1.2": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.9.tgz#15f529d247f1ede1824f7e7acdaa192d5f28071e" + integrity sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*", "@types/node@^20.8.5": version "20.8.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.5.tgz#13352ae1f80032171616910e8aba2e3e52e57d96" @@ -2174,6 +2274,16 @@ dependencies: undici-types "~5.25.1" +"@types/offscreencanvas@~2019.3.0": + version "2019.3.0" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" + integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== + +"@types/offscreencanvas@~2019.7.0": + version "2019.7.3" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz#90267db13f64d6e9ccb5ae3eac92786a7c77a516" + integrity sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A== + "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" @@ -2237,6 +2347,11 @@ resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.4.tgz#fedc3e5b15c26dc18faae96bf1317487cb3658cf" integrity sha512-2L9ifAGl7wmXwP4v3pN4p2FLhD0O1qsJpvKmNin5VA8+UvNVb447UDaAEV6UdrkA+m/Xs58U1RFps44x6TFsVQ== +"@types/seedrandom@^2.4.28": + version "2.4.34" + resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.34.tgz#c725cd0fc0442e2d3d0e5913af005686ffb7eb99" + integrity sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A== + "@types/semver@^7.3.12", "@types/semver@^7.5.0": version "7.5.3" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.3.tgz#9a726e116beb26c24f1ccd6850201e1246122e04" @@ -2627,6 +2742,11 @@ "@webassemblyjs/ast" "1.11.6" "@xtuc/long" "4.2.2" +"@webgpu/types@0.1.30": + version "0.1.30" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.30.tgz#b6406dc4a1c1e0d469028ceb30ddffbbd2fa706c" + integrity sha512-9AXJSmL3MzY8ZL//JjudA//q+2kBRGhLBFpkdGksWIuxrMy81nFrCzj2Am+mbh8WoU6rXmv7cY5E3rdlyru2Qg== + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -2801,7 +2921,7 @@ arg@^5.0.2: resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== -argparse@^1.0.7: +argparse@^1.0.10, argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== @@ -3606,6 +3726,11 @@ core-js-pure@^3.23.3: resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.33.0.tgz#938a28754b4d82017a7a8cbd2727b1abecc63591" integrity sha512-FKSIDtJnds/YFIEaZ4HszRX7hkxGpNKM7FC9aJ9WLJbSd3lD4vOltFuVIBLR8asSx9frkTSqL0dw90SKQxgKrg== +core-js@3.29.1: + version "3.29.1" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.29.1.tgz#40ff3b41588b091aaed19ca1aa5cb111803fa9a6" + integrity sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw== + core-js@^3.19.2: version "3.33.0" resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.33.0.tgz#70366dbf737134761edb017990cf5ce6c6369c40" @@ -6714,6 +6839,11 @@ lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +long@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -6968,6 +7098,13 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +node-fetch@~2.6.1: + version "2.6.13" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.13.tgz#a20acbbec73c2e09f9007de5cda17104122e0010" + integrity sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -8259,6 +8396,11 @@ react-scripts@5.0.1: optionalDependencies: fsevents "^2.3.2" +react-webcam@^7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/react-webcam/-/react-webcam-7.2.0.tgz#64141c4c7bdd3e956620500187fa3fcc77e1fd49" + integrity sha512-xkrzYPqa1ag2DP+2Q/kLKBmCIfEx49bVdgCCCcZf88oF+0NPEbkwYk3/s/C7Zy0mhM8k+hpdNkBLzxg8H0aWcg== + react@^18.2.0: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" @@ -8348,7 +8490,7 @@ regenerate@^1.4.2: resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.4.2.tgz#b9346d8827e8f5a32f7ba29637d398b69014848a" integrity sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A== -regenerator-runtime@^0.13.9: +regenerator-runtime@^0.13.5, regenerator-runtime@^0.13.9: version "0.13.11" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== @@ -8587,6 +8729,13 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" +scatter-gl@^0.0.13: + version "0.0.13" + resolved "https://registry.yarnpkg.com/scatter-gl/-/scatter-gl-0.0.13.tgz#7bc3ded68ef726fe581529efbeb4beddf0a92c8f" + integrity sha512-304ftr2ND7RSZ5tmnnPNEGc6D9DH+xSP5OJU6DrwvDLA19We8icrVWmdpJbORC9hh0cZ/gYsn+/6H+WONdUGeQ== + dependencies: + three "0.125" + scheduler@^0.23.0: version "0.23.0" resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.0.tgz#ba8041afc3d30eb206a487b6b384002e4e61fdfe" @@ -8631,6 +8780,11 @@ schema-utils@^4.0.0: ajv-formats "^2.1.1" ajv-keywords "^5.1.0" +seedrandom@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7" + integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg== + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -8990,7 +9144,7 @@ string.prototype.trimstart@^1.0.7: define-properties "^1.2.0" es-abstract "^1.22.1" -string_decoder@^1.1.1: +string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -9319,6 +9473,11 @@ thenify-all@^1.0.0: dependencies: any-promise "^1.0.0" +three@0.125: + version "0.125.2" + resolved "https://registry.yarnpkg.com/three/-/three-0.125.2.tgz#dcba12749a2eb41522e15212b919cd3fbf729b12" + integrity sha512-7rIRO23jVKWcAPFdW/HREU2NZMGWPBZ4XwEMt0Ak0jwLUKVJhcKM55eCBWyGZq/KiQbeo1IeuAoo/9l2dzhTXA== + throat@^6.0.1: version "6.0.2" resolved "https://registry.yarnpkg.com/throat/-/throat-6.0.2.tgz#51a3fbb5e11ae72e2cf74861ed5c8020f89f29fe" @@ -9390,6 +9549,11 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + tryer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" @@ -9415,6 +9579,11 @@ tsconfig-paths@^3.14.2: minimist "^1.2.6" strip-bom "^3.0.0" +tslib@2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" + integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== + tslib@^1.8.1: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -9718,6 +9887,11 @@ web-vitals@^2.1.0: resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-2.1.4.tgz#76563175a475a5e835264d373704f9dde718290c" integrity sha512-sVWcwhU5mX6crfI5Vd2dC4qchyTqxV8URinzt25XqVh+bHEPGH4C3NPrNionCP7Obx59wrYEbNlw4Z8sjALzZg== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + webidl-conversions@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad" @@ -9870,6 +10044,14 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + whatwg-url@^7.0.0: version "7.1.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06" @@ -10200,7 +10382,7 @@ yargs-parser@^20.2.2: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== -yargs@^16.2.0: +yargs@^16.0.3, yargs@^16.2.0: version "16.2.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== From 5fdc2e42e815a913a9956f31d8d76fc2421ca97d Mon Sep 17 00:00:00 2001 From: gyuhyuk Date: Thu, 23 Nov 2023 01:07:08 +0900 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20react-webcam=EC=97=90=EC=84=9C=20op?= =?UTF-8?q?enVidu=EB=A1=9C=20=EB=B3=80=EA=B2=BD=ED=95=98=EC=97=AC=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + src/App.tsx | 2 +- src/assets/pages/Game/GameRoom.js | 74 ---- src/assets/room/openvidu/GameRoom.js | 318 ++++++++++++++++++ src/assets/room/openvidu/UserVideo.css | 26 ++ .../room/openvidu/UserVideoComponent.js | 22 ++ src/assets/room/openvidu/ovVideo.js | 55 +++ .../params.tsx => room/openvidu/params.ts} | 0 .../room/openvidu/registerServiceWorker.js | 117 +++++++ .../{pages/Game => room/openvidu}/util.js | 0 src/index.tsx | 3 + yarn.lock | 74 +++- 12 files changed, 616 insertions(+), 76 deletions(-) delete mode 100644 src/assets/pages/Game/GameRoom.js create mode 100644 src/assets/room/openvidu/GameRoom.js create mode 100644 src/assets/room/openvidu/UserVideo.css create mode 100644 src/assets/room/openvidu/UserVideoComponent.js create mode 100644 src/assets/room/openvidu/ovVideo.js rename src/assets/{pages/Game/params.tsx => room/openvidu/params.ts} (100%) create mode 100644 src/assets/room/openvidu/registerServiceWorker.js rename src/assets/{pages/Game => room/openvidu}/util.js (100%) diff --git a/package.json b/package.json index 580c6c6..2966182 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@types/react": "^18.0.0", "@types/react-dom": "^18.2.13", "axios": "^1.5.1", + "openvidu-browser": "^2.29.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-hook-form": "^7.47.0", diff --git a/src/App.tsx b/src/App.tsx index 9f88480..bdbfbe3 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,7 +9,7 @@ import SquatGameGuide from './assets/modal/SquatGameGuide'; import Dashboard from './assets/pages/Dashboard'; import MainHeader from './assets/component/MainHeader'; -import GameRoom from './assets/pages/Game/GameRoom'; +import GameRoom from './assets/room/openvidu/GameRoom'; function App() { return ( diff --git a/src/assets/pages/Game/GameRoom.js b/src/assets/pages/Game/GameRoom.js deleted file mode 100644 index 4d6ec35..0000000 --- a/src/assets/pages/Game/GameRoom.js +++ /dev/null @@ -1,74 +0,0 @@ -import { useRef, useEffect } from 'react'; -import '@tensorflow/tfjs-backend-webgl'; -import * as poseDetection from '@tensorflow-models/pose-detection'; -import Webcam from 'react-webcam'; -import * as tf from '@tensorflow/tfjs-core'; - -const GameRoom = () => { - const webcamRef = useRef(null); - // const canvasRef = useRef(null); - - const runBlazePose = async () => { - const detector = await poseDetection.createDetector( - poseDetection.SupportedModels.BlazePose, - { - runtime: 'tfjs', - }, - ); - - setInterval(() => { - detect(detector); - }, 20); - }; - - useEffect(() => { - const initializeTensorflow = async () => { - await tf.ready(); - runBlazePose(); - }; - - initializeTensorflow(); - }, []); - - const detect = async (net) => { - if ( - typeof webcamRef.current !== 'undefined' && - webcamRef.current !== null && - webcamRef.current.video.readyState === 4 - ) { - const video = webcamRef.current.video; - const videoWidth = video.videoWidth; - const videoHeight = video.videoHeight; - - webcamRef.current.video.width = videoWidth; - webcamRef.current.video.height = videoHeight; - - const pose = await net.estimatePoses(video); - console.log(pose); - // drawCanvas(pose, videoWidth, videoHeight, canvasRef); - } - }; - - window.requestAnimationFrame(runBlazePose); - return ( -
- -
- ); -}; - -export default GameRoom; diff --git a/src/assets/room/openvidu/GameRoom.js b/src/assets/room/openvidu/GameRoom.js new file mode 100644 index 0000000..b483153 --- /dev/null +++ b/src/assets/room/openvidu/GameRoom.js @@ -0,0 +1,318 @@ +import { OpenVidu } from 'openvidu-browser'; +import { useRef, useEffect, useState, useCallback } from 'react'; +import '@tensorflow/tfjs-backend-webgl'; +import UserVideoComponent from './UserVideoComponent'; +import axios from 'axios'; + +const APPLICATION_SERVER_URL = + process.env.NODE_ENV === 'production' ? '' : 'https://demos.openvidu.io/'; + +export default function GameRoom() { + const [mySessionId, setMySessionId] = useState('test'); + const [myUserName, setMyUserName] = useState( + `Participant${Math.floor(Math.random() * 100)}`, + ); + const [session, setSession] = useState(undefined); + const [mainStreamManager, setMainStreamManager] = useState(undefined); + const [publisher, setPublisher] = useState(undefined); + const [subscribers, setSubscribers] = useState([]); + const [currentVideoDevice, setCurrentVideoDevice] = useState(null); + + const OV = useRef(new OpenVidu()); + + const handleChangeSessionId = useCallback((e) => { + setMySessionId(e.target.value); + }, []); + + const handleChangeUserName = useCallback((e) => { + setMyUserName(e.target.value); + }, []); + + const handleMainVideoStream = useCallback( + (stream) => { + if (mainStreamManager !== stream) { + setMainStreamManager(stream); + } + }, + [mainStreamManager], + ); + + const joinSession = useCallback(() => { + const mySession = OV.current.initSession(); + + mySession.on('streamCreated', (event) => { + const subscriber = mySession.subscribe(event.stream, undefined); + setSubscribers((subscribers) => [...subscribers, subscriber]); + }); + + mySession.on('streamDestroyed', (event) => { + deleteSubscriber(event.stream.streamManager); + }); + + mySession.on('exception', (exception) => { + console.warn(exception); + }); + + setSession(mySession); + }, []); + + useEffect(() => { + if (session) { + // Get a token from the OpenVidu deployment + getToken().then(async (token) => { + try { + await session.connect(token, { clientData: myUserName }); + + let publisher = await OV.current.initPublisherAsync(undefined, { + audioSource: undefined, + videoSource: undefined, + publishAudio: true, + publishVideo: true, + resolution: '640x480', + frameRate: 30, + insertMode: 'APPEND', + mirror: false, + }); + + session.publish(publisher); + + const devices = await OV.current.getDevices(); + const videoDevices = devices.filter( + (device) => device.kind === 'videoinput', + ); + const currentVideoDeviceId = publisher.stream + .getMediaStream() + .getVideoTracks()[0] + .getSettings().deviceId; + const currentVideoDevice = videoDevices.find( + (device) => device.deviceId === currentVideoDeviceId, + ); + + setMainStreamManager(publisher); + setPublisher(publisher); + setCurrentVideoDevice(currentVideoDevice); + } catch (error) { + console.log( + 'There was an error connecting to the session:', + error.code, + error.message, + ); + } + }); + } + }, [session, myUserName]); + + const leaveSession = useCallback(() => { + // Leave the session + if (session) { + session.disconnect(); + } + + // Reset all states and OpenVidu object + OV.current = new OpenVidu(); + setSession(undefined); + setSubscribers([]); + setMySessionId('SessionA'); + setMyUserName('Participant' + Math.floor(Math.random() * 100)); + setMainStreamManager(undefined); + setPublisher(undefined); + }, [session]); + + const switchCamera = useCallback(async () => { + try { + const devices = await OV.current.getDevices(); + const videoDevices = devices.filter( + (device) => device.kind === 'videoinput', + ); + + if (videoDevices && videoDevices.length > 1) { + const newVideoDevice = videoDevices.filter( + (device) => device.deviceId !== currentVideoDevice.deviceId, + ); + + if (newVideoDevice.length > 0) { + const newPublisher = OV.current.initPublisher(undefined, { + videoSource: newVideoDevice[0].deviceId, + publishAudio: true, + publishVideo: true, + mirror: true, + }); + + if (session) { + await session.unpublish(mainStreamManager); + await session.publish(newPublisher); + setCurrentVideoDevice(newVideoDevice[0]); + setMainStreamManager(newPublisher); + setPublisher(newPublisher); + } + } + } + } catch (e) { + console.error(e); + } + }, [currentVideoDevice, session, mainStreamManager]); + + const deleteSubscriber = useCallback((streamManager) => { + setSubscribers((prevSubscribers) => { + const index = prevSubscribers.indexOf(streamManager); + if (index > -1) { + const newSubscribers = [...prevSubscribers]; + newSubscribers.splice(index, 1); + return newSubscribers; + } else { + return prevSubscribers; + } + }); + }, []); + + useEffect(() => { + const handleBeforeUnload = (event) => { + leaveSession(); + }; + window.addEventListener('beforeunload', handleBeforeUnload); + + return () => { + window.removeEventListener('beforeunload', handleBeforeUnload); + }; + }, [leaveSession]); + + /** + * -------------------------------------------- + * GETTING A TOKEN FROM YOUR APPLICATION SERVER + * -------------------------------------------- + * The methods below request the creation of a Session and a Token to + * your application server. This keeps your OpenVidu deployment secure. + * + * In this sample code, there is no user control at all. Anybody could + * access your application server endpoints! In a real production + * environment, your application server must identify the user to allow + * access to the endpoints. + * + * Visit https://docs.openvidu.io/en/stable/application-server to learn + * more about the integration of OpenVidu in your application server. + */ + const getToken = useCallback(async () => { + return createSession(mySessionId).then((sessionId) => + createToken(sessionId), + ); + }, [mySessionId]); + + const createSession = async (sessionId) => { + const response = await axios.post( + APPLICATION_SERVER_URL + 'api/sessions', + { customSessionId: sessionId }, + { + headers: { 'Content-Type': 'application/json' }, + }, + ); + return response.data; // The sessionId + }; + + const createToken = async (sessionId) => { + const response = await axios.post( + APPLICATION_SERVER_URL + 'api/sessions/' + sessionId + '/connections', + {}, + { + headers: { 'Content-Type': 'application/json' }, + }, + ); + return response.data; // The token + }; + return ( +
+ {session === undefined ? ( +
+
+ OpenVidu logo +
+
+

Join a video session

+
+

+ + +

+

+ + +

+

+ +

+
+
+
+ ) : null} + + {session !== undefined ? ( +
+
+

{mySessionId}

+ + +
+ + {mainStreamManager !== undefined ? ( +
+ +
+ ) : null} +
+ {publisher !== undefined ? ( +
handleMainVideoStream(publisher)} + > + +
+ ) : null} + {subscribers.map((sub, i) => ( +
handleMainVideoStream(sub)} + > + {sub.id} + +
+ ))} +
+
+ ) : null} +
+ ); +} diff --git a/src/assets/room/openvidu/UserVideo.css b/src/assets/room/openvidu/UserVideo.css new file mode 100644 index 0000000..ca667a9 --- /dev/null +++ b/src/assets/room/openvidu/UserVideo.css @@ -0,0 +1,26 @@ +video { + width: 100px; + height: 100px; + float: left; + cursor: pointer; +} +.streamcomponent div { + position: absolute; + background: #f8f8f8; + padding-left: 5px; + padding-right: 5px; + color: #777777; + font-weight: bold; + border-bottom-right-radius: 4px; +} +p { + margin: 0; +} + +#video-container { + visibility: hidden; +} + +/* + visibility : hidden -> 카메라 2개 뜨는것 때문에 일시적으로 해놓음. +*/ diff --git a/src/assets/room/openvidu/UserVideoComponent.js b/src/assets/room/openvidu/UserVideoComponent.js new file mode 100644 index 0000000..3e24e31 --- /dev/null +++ b/src/assets/room/openvidu/UserVideoComponent.js @@ -0,0 +1,22 @@ +import OpenViduVideoComponent from './ovVideo'; +import './UserVideo.css'; + +export default function UserVideoComponent({ streamManager }) { + const getNicknameTag = () => { + // Gets the nickName of the user + return JSON.parse(streamManager.stream.connection.data).clientData; + }; + + return ( +
+ {streamManager !== undefined ? ( +
+ +
+

{getNicknameTag()}

+
+
+ ) : null} +
+ ); +} diff --git a/src/assets/room/openvidu/ovVideo.js b/src/assets/room/openvidu/ovVideo.js new file mode 100644 index 0000000..2196354 --- /dev/null +++ b/src/assets/room/openvidu/ovVideo.js @@ -0,0 +1,55 @@ +import { useRef, useEffect } from 'react'; +import * as poseDetection from '@tensorflow-models/pose-detection'; +import * as tf from '@tensorflow/tfjs-core'; + +export default function OpenViduVideoComponent({ streamManager }) { + const videoRef = useRef(); + + const runBlazePose = async () => { + const detector = await poseDetection.createDetector( + poseDetection.SupportedModels.BlazePose, + { + runtime: 'tfjs', + }, + ); + + setInterval(() => { + detect(detector); + }, 20); + }; + + const detect = async (net) => { + if ( + typeof videoRef.current !== 'undefined' && + videoRef.current !== null && + videoRef.current.readyState === 4 + ) { + const video = videoRef.current; + const videoWidth = video.videoWidth; + const videoHeight = video.videoHeight; + + video.width = videoWidth; + video.height = videoHeight; + + const pose = await net.estimatePoses(video); + // console.log(pose); pose 콘솔 확인 + } + }; + + useEffect(() => { + const initializeTensorflow = async () => { + await tf.ready(); + runBlazePose(); + }; + + initializeTensorflow(); + }, []); + + useEffect(() => { + if (streamManager && videoRef.current) { + streamManager.addVideoElement(videoRef.current); + } + }, [streamManager]); + + return