8000 feat(state) speech support for player2 by lorisj · Pull Request #210 · moeru-ai/airi · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

feat(state) speech support for player2 #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.ja-JP.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ pnpm -F @proj-airi/stage-tamagotchi dev

## サポートされているLLM APIプロバイダー([xsai](https://github.com/moeru-ai/xsai)によって提供)

- [x] [Player2](https://player2.game/)
- [x] [OpenRouter](https://openrouter.ai/)
- [x] [vLLM](https://github.com/vllm-project/vllm)
- [x] [SGLang](https://github.com/sgl-project/sglang)
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ npx bumpp --no-commit --no-tag
```

## Supported the following LLM API Providers (powered by [xsai](https://github.com/moeru-ai/xsai))

- [x] [Player2](https://player2.game/)
- [x] [OpenRouter](https://openrouter.ai/)
- [x] [vLLM](https://github.com/vllm-project/vllm)
- [x] [SGLang](https://github.com/sgl-project/sglang)
Expand Down
1 change: 1 addition & 0 deletions README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ pnpm -F @proj-airi/stage-tamagotchi dev

## 原生支持的 LLM API 提供商列表(由 [xsai](https://github.com/moeru-ai/xsai) 驱动)

- [x] [Player2](https://player2.game/)
- [x] [OpenRouter](https://openrouter.ai/)
- [x] [vLLM](https://github.com/vllm-project/vllm)
- [x] [SGLang](https://github.com/sgl-project/sglang)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ async function generateTestSpeech() {
if (audioUrl.value) {
stopTestAudio()
}

const input = useSSML.value
? ssmlText.value
: speechStore.generateSSML(testText.value, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
Expand Down
2 changes: 1 addition & 1 deletion apps/stage-web/src/pages/settings/modules/speech.vue
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ async function generateTestSpeech() {

const input = useSSML.value
? ssmlText.value
: speechStore.generateSSML(testText.value, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
: speechStore.supportsSSML ? speechStore.generateSSML(testText.value, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value }) : testText.value

const response = await generateSpeech({
...provider.speech(activeSpeechModel.value, providerConfig),
Expand Down
133 changes: 133 additions & 0 deletions apps/stage-web/src/pages/settings/providers/player2-speech.vue
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
<script setup lang="ts">
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers'
import type { UnElevenLabsOptions } from 'unspeech'

import {
SpeechPlayground,
SpeechProviderSettings,
} from '@proj-airi/stage-ui/components'
import { useProvidersStore, useSpeechStore } from '@proj-airi/stage-ui/stores'
import { FieldRange } from '@proj-airi/ui'
import { computed, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'

const providerId = 'player2-speech'
const defaultModel = 'v1'

const speedRatio = ref<number>(1.0)

const speechStore = useSpeechStore()
const providersStore = useProvidersStore()
const { t } = useI18n()

// Get available voices for ElevenLabs
const availableVoices = computed(() => {
return speechStore.availableVoices[providerId] || []
})

// Generate speech with ElevenLabs-specific parameters
async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean) {
const provider = providersStore.getProviderInstance(providerId) as SpeechProviderWithExtraOptions<string, UnElevenLabsOptions>
if (!provider) {
throw new Error('Failed to initialize speech provider')
}

// Get provider configuration
const providerConfig = providersStore.getProviderConfig(providerId)

// Get model from configuration or use default
const model = providerConfig.model as string | undefined || defaultModel

// ElevenLabs doesn't need SSML conversion, but if SSML is provided, use it directly
return await speechStore.speech(
provider,
model,
input,
voiceId,
{
...providerConfig,
},
)
}

const hasPlayer2 = ref(true)

onMounted(async () => {
const providerConfig = providersStore.getProviderConfig(providerId)
const providerMetadata = providersStore.getProviderMetadata(providerId)
if (await providerMetadata.validators.validateProviderConfig(providerConfig)) {
await speechStore.loadVoicesForProvider(providerId)
}
else {
console.error('Failed to validate provider config', providerConfig)
}
try {
const res = await fetch(`http://localhost:4315/v1/health`, {
method: 'GET',
headers: {
'player2-game-key': 'airi',
},
})
hasPlayer2.value = res.status === 200
}
catch (e) {
console.error(e)
hasPlayer2.value = false
}
})

watch(speedRatio, async () => {
const providerConfig = providersStore.getProviderConfig(providerId)
providerConfig.speed = speedRatio.value
})
</script>

<template>
<div v-if="!hasPlayer2" style="color: red; margin-bottom: 1rem;">
<div>
Please download and run the Player2 App:
<a href="https://player2.game" target="_blank" rel="noopener noreferrer">
https://player2.game
</a>

<div>
After downloading, if you still are having trouble, please reach out to us on Discord:
<a href="https://player2.game/discord" target="_blank" rel="noopener noreferrer">
https://player2.game/discord
</a>.
</div>
</div>
</div>
<SpeechProviderSettings
:provider-id="providerId"
:default-model="defaultModel"
>
<template #voice-settings>
<!-- Speed control - common to most providers -->
<FieldRange
v-model="speedRatio"
:label="t('settings.pages.providers.provider.common.fields.field.speed.label')"
:description="t('settings.pages.providers.provider.common.fields.field.speed.description')"
:min="0.5"
:max="5.0" :step="0.01"
/>
</template>

<!-- Replace the default playground with our standalone component -->
<template #playground>
<SpeechPlayground
:available-voices="availableVoices"
:generate-speech="handleGenerateSpeech"
:api-key-configured="true"
default-text="Hello! This is a test of the Player 2 voice synthesis."
/>
</template>
</SpeechProviderSettings>
</template>

<route lang="yaml">
meta:
layout: settings
stageTransition:
name: slide
</route>
Original file line number Diff line number Diff line change
@@ -1,40 +1,54 @@
<script setup lang="ts">
import type { RemovableRef } from '@vueuse/shared'

import {
ProviderBaseUrlInput,
ProviderSettingsContainer,
ProviderSettingsLayout,
} from '@proj-airi/stage-ui/components'
import { useProvidersStore } from '@proj-airi/stage-ui/stores'
import { storeToRefs } from 'pinia'
import { computed, onMounted, watch } from 'vue'
import { computed, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'

const { t } = useI18n()
const router = useRouter()
const providersStore = useProvidersStore()
const { providers } = storeToRefs(providersStore)
const { providers } = storeToRefs(providersStore) as { providers: RemovableRef<Record<string, any>> }

// Get provider metadata
const providerId = 'player2-api'
const providerId = 'player2'
const providerMetadata = computed(() => providersStore.getProviderMetadata(providerId))

const baseUrl = computed({
get: () => providers.value[providerId]?.baseUrl as string || '',
get: () => providers.value[providerId]?.baseUrl || '',
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}

providers.value[providerId].baseUrl = value
},
})
const hasPlayer2 = ref(true)

onMounted(() => {
// Initialize provider if it doesn't exist
onMounted(async () => {
providersStore.initializeProvider(providerId)
baseUrl.value = providers.value[providerId]?.baseUrl || ''

// Initialize refs with current values
baseUrl.value = providers.value[providerId]?.baseUrl as string || ''
try {
const res = await fetch(`${baseUrl.value}health`, {
method: 'GET',
headers: {
'player2-game-key': 'airi',
},
})
hasPlayer2.value = res.status === 200
}
catch (e) {
console.error(e)
hasPlayer2.value = false
}
})

// Watch settings and update the provider configuration
Expand All @@ -53,8 +67,25 @@ function handleResetSettings() {
</script>

<template>
<div v-if="!hasPlayer2" style="color: red; margin-bottom: 1rem;">
<div>
Please download and run the Player2 App:
<a href="https://player2.game" target="_blank" rel="noopener noreferrer">
https://player2.game
</a>

<div>
After downloading, if you still are having trouble, please reach out to us on Discord:
<a href="https://player2.game/discord" target="_blank" rel="noopener noreferrer">
https://player2.game/discord
</a>.
</div>
</div>
</div>

<ProviderSettingsLayout
:provider-name="providerMetadata?.localizedName" :provider-icon="providerMetadata?.icon"
:provider-name="providerMetadata?.localizedName"
:provider-icon="providerMetadata?.icon"
:on-back="() => router.back()"
>
<ProviderSettingsContainer>
Expand Down
10 changes: 8 additions & 2 deletions packages/stage-ui/src/stores/modules/consciousness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,18 @@ export const useConsciousnessStore = defineStore('consciousness', () => {
await loadModelsForProvider(newProvider)
resetModelSelection()

if (newProvider === 'player2-api') {
if (newProvider === 'player2') {
// Ping heal check every 60 seconds if Player2 is being used
player2Interval = setInterval(() => {
// eslint-disable-next-line no-console
console.log('Sending Player2 Health check if it is being used')
fetch('http://localhost:4315/v1/health').catch(() => {})
fetch(`localhost:4315/v1/health`, {
method: 'GET',
headers: {
'player2-game-key': 'airi',
},
})
.catch(() => {})
}, 60_000)
}
else {
Expand Down
2 changes: 1 addition & 1 deletion packages/stage-ui/src/stores/modules/speech.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export const useSpeechStore = defineStore('speech', () => {

// Helper function to determine if a provider is a speech provider
function isSpeechProvider(providerId: string): boolean {
return ['elevenlabs', 'microsoft-speech', 'azure-speech', 'google', 'amazon', 'alibaba-cloud-model-studio', 'volcengine'].includes(providerId)
return ['elevenlabs', 'microsoft-speech', 'azure-speech', 'google', 'amazon', 'alibaba-cloud-model-studio', 'volcengine', 'player2-speech'].includes(providerId)
}

async function loadVoicesForProvider(provider: string) {
Expand Down
Loading
0