8000 Switch WebGLMicrophone implementation from ScriptProcessor to AudioWo… · uezo/ChatdollKit@299b3f0 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 299b3f0

Browse files
uezouezo
committed
Switch WebGLMicrophone implementation from ScriptProcessor to AudioWorklet
- Updated `Plugins/WebGLMicrophone.jslib` to use AudioWorklet instead of the deprecated ScriptProcessor. - This change enables microphone audio processing to run on a separate thread from the main thread, improving performance and future compatibility. - To use this feature, please copy `Plugins/CopyThisJSToStreamingAssets/WebGLMicrophoneProcessor.js` to the `StreamingAssets` folder.
1 parent af3a3fc commit 299b3f0

File tree

4 files changed

+105
-56
lines changed

4 files changed

+105
-56
lines changed

‎Plugins/CopyThisJSToStreamingAssets.meta

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Plugins/CopyThisJSToStreamingAssets/WebGLMicrophoneProcessor.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
class WebGLMicrophoneProcessor extends AudioWorkletProcessor {
2+
process(inputs) {
3+
const input = inputs[0];
4+
if (input && input[0].length > 0) {
5+
this.port.postMessage(input[0]);
6+
}
7+
return true;
8+
}
9+
}
10+
11+
registerProcessor("webgl-microphone-processor", WebGLMicrophoneProcessor);

‎Plugins/CopyThisJSToStreamingAssets/WebGLMicrophoneProcessor.js.meta

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Plugins/WebGLMicrophone.jslib

Lines changed: 79 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,88 @@
11
mergeInto(LibraryManager.library, {
2-
InitWebGLMicrophone: function(targetObjectNamePtr) {
3-
// Initialize webGLMicrophone
4-
document.webGLMicrophone = new Object();
5-
document.webGLMicrophone.isRecording = 0;
6-
document.webGLMicrophone.targetObjectName = UTF8ToString(targetObjectNamePtr);
7-
document.webGLMicrophone.audioContext = new (window.AudioContext || window.webkitAudioContext)();
8-
document.webGLMicrophone.source = null;
9-
document.webGLMicrophone.scriptNode = null;
2+
InitWebGLMicrophone: function(targetObjectNamePtr) {
3+
var mic = document.webGLMicrophone = {};
4+
mic.isRecording = 0;
5+
mic.targetObjectName = UTF8ToString(targetObjectNamePtr);
6+
mic.audioContext = new (window.AudioContext || window.webkitAudioContext)();
7+
mic.source = null;
8+
mic.workletNode = null;
9+
mic.processorModuleUrl = "StreamingAssets/WebGLMicrophoneProcessor.js";
10+
mic._buffer = [];
11+
mic._bufferSize = 4096;
1012

11-
// Observe the state of audio context for microphone and resume if disabled
12-
setInterval(function() {
13-
if (document.webGLMicrophone.audioContext.state === "suspended" || document.webGLMicrophone.audioContext.state === "interrupted") {
14-
console.log("Resuming AudioContext: " + document.webGLMicrophone.audioContext.state);
15-
document.webGLMicrophone.audioContext.resume();
16-
}
17-
}, 300);
18-
},
13+
setInterval(function() {
14+
var ac = mic.audioContext;
15+
if (ac.state === "suspended" || ac.state === "interrupted") {
16+
console.log("Resuming AudioContext:", ac.state);
17+
ac.resume();
18+
}
19+
}, 300);
20+
},
21+
22+
StartWebGLMicrophone: function() {
23+
var mic = document.webGLMicrophone;
24+
mic.isRecording = 1;
25+
26+
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
27+
console.error("getUserMedia not supported");
28+
mic.isRecording = 0;
29+
return;
30+
}
1931

20-
StartWebGLMicrophone: function() {
21-
document.webGLMicrophone.isRecording = 1;
32+
navigator.mediaDevices.getUserMedia({
33+
audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 }
34+
})
35+
.then(function(stream) {
36+
var ac = mic.audioContext;
37+
ac.audioWorklet.addModule(mic.processorModuleUrl)
38+
.then(function() {
39+
mic.source = ac.createMediaStreamSource(stream);
40+
mic.workletNode = new AudioWorkletNode(ac, "webgl-microphone-processor");
2241

23-
if (navigator.mediaDevices.getUserMedia) {
24-
navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1} })
25-
.then(function(stream) {
26-
// Setup nodes
27-
var audioContext = document.webGLMicrophone.audioContext;
28-
var source = audioContext.createMediaStreamSource(stream);
29-
var scriptNode = audioContext.createScriptProcessor(4096, 1, 1);
30-
scriptNode.onaudioprocess = function (event) {
31-
SendMessage(document.webGLMicrophone.targetObjectName, "SetSamplingData", event.inputBuffer.getChannelData(0).join(','));
32-
};
33-
// Connect nodes;
34-
source.connect(scriptNode);
35-
scriptNode.connect(audioContext.destination);
42+
mic.workletNode.port.onmessage = function(event) {
43+
var float32Array = event.data;
44+
for (var i = 0; i < float32Array.length; i++) {
45+
mic._buffer.push(float32Array[i]);
46+
}
47+
if (mic._buffer.length >= mic._bufferSize) {
48+
var chunk = mic._buffer.slice(0, mic._bufferSize);
49+
mic._buffer = mic._buffer.slice(mic._bufferSize);
50+
var csv = Array.prototype.join.call(chunk, ",");
51+
SendMessage(mic.targetObjectName, "SetSamplingData", csv);
52+
}
53+
};
3654

37-
document.webGLMicrophone.scriptNode = scriptNode;
38-
document.webGLMicrophone.source = source;
55+
mic.source.connect(mic.workletNode);
56+
mic.workletNode.connect(ac.destination);
3957

40-
console.log("WebGLMicrophone started recording");
41-
})
42-
.catch(function(err) {
43-
console.log("Failed in GetUserMedia: " + error);
44-
document.webGLMicrophone.isRecording = 0;
45-
});
46-
}
47-
},
58+
console.log("WebGLMicrophone started recording");
59+
})
60+
.catch(function(err) {
61+
console.error("Failed to load AudioWorklet module:", err);
62+
mic.isRecording = 0;
63+
});
64+
})
65+
.catch(function(err) {
66+
console.error("Failed in getUserMedia:", err);
67+
mic.isRecording = 0;
68+
});
69+
},
4870

49-
EndWebGLMicrophone: function() {
50-
console.log("EndWebGLMicrophone");
51-
if (document.webGLMicrophone.source != null) {
52-
document.webGLMicrophone.source.disconnect(document.webGLMicrophone.scriptNode);
53-
document.webGLMicrophone.source = null;
54-
}
55-
if (document.webGLMicrophone.scriptNode != null) {
56-
document.webGLMicrophone.scriptNode.disconnect();
57-
document.webGLMicrophone.scriptNode = null;
58-
}
59-
document.webGLMicrophone.isRecording = 0;
60-
},
71+
EndWebGLMicrophone: function() {
72+
var mic = document.webGLMicrophone;
73+
console.log("EndWebGLMicrophone");
74+
if (mic.source && mic.workletNode) {
75+
mic.source.disconnect(mic.workletNode);
76+
mic.workletNode.disconnect();
77+
}
78+
mic.source = null;
79+
mic.workletNode = null;
80+
mic._buffer = [];
81+
mic.isRecording = 0;
82+
},
6183

62-
IsWebGLMicrophoneRecording: function() {
63-
return document.webGLMicrophone.isRecording == 1;
64-
},
84+
IsWebGLMicrophoneRecording: function() {
85+
return document.webGLMicrophone &&
86+
document.webGLMicrophone.isRecording === 1;
87+
},
6588
});

0 commit comments

Comments
 (0)
0