talk/internal/langdetect/detect.go

// Package langdetect guesses whether an announcement is German or English.
// It is a pure function over two small word lists, deliberately without any
// ML: announcements are one or two sentences, and the only question is which
// voice should read them.
package langdetect

import (
	"strings"
	"unicode"
)

// Detect returns "de" or "en" for text. Umlauts and ß count as strong German
// signals; otherwise common-word hits decide. English wins only with strictly
// more English than German hits; ties and undecidable text (empty, numbers,
// punctuation) return fallback, which callers take from the configured
// default language.
func Detect(text, fallback string) string {
	var de, en int
	for _, r := range text {
		switch r {
		case 'ä', 'ö', 'ü', 'ß', 'Ä', 'Ö', 'Ü':
			de++
		}
	}

	for _, word := range strings.FieldsFunc(text, func(r rune) bool {
		return !unicode.IsLetter(r) && r != '\'' && r != '’'
	}) {
		word = strings.ToLower(word)
		if _, ok := germanWords[word]; ok {
			de++
		}
		if _, ok := englishWords[word]; ok {
			en++
		}
	}

	switch {
	case en > de:
		return "en"
	case de > en:
		return "de"
	default:
		return fallback
	}
}

// germanWords are common German words that do not occur in English
// (ambiguous ones such as "was" and "in" are left out on purpose).
var germanWords = map[string]struct{}{
	"der": {}, "die": {}, "das": {}, "und": {}, "nicht": {}, "ist": {},
	"ein": {}, "eine": {}, "einen": {}, "einem": {}, "einer": {},
	"mit": {}, "für": {}, "auf": {}, "von": {}, "vom": {}, "zum": {},
	"zur": {}, "den": {}, "dem": {}, "des": {}, "wir": {}, "ihr": {},
	"sie": {}, "auch": {}, "als": {}, "wie": {}, "bei": {},
	"aus": {}, "nach": {}, "noch": {}, "nur": {}, "schon": {},
	"wenn": {}, "dann": {}, "aber": {}, "oder": {}, "doch": {},
	"mehr": {}, "sehr": {}, "über": {}, "unter": {}, "jetzt": {},
	"hier": {}, "heute": {}, "morgen": {}, "danke": {}, "bitte": {},
	"erfolgreich": {}, "fehlgeschlagen": {}, "fertig": {},
	"gestartet": {}, "beendet": {}, "abgeschlossen": {}, "warten": {},
	"läuft": {}, "kannst": {}, "muss": {}, "soll": {},
	"kein": {}, "keine": {}, "wieder": {}, "werden": {}, "wurde": {},
	"wurden": {}, "haben": {}, "geht": {}, "machen": {},
}

// englishWords are common English words that do not occur in German.
var englishWords = map[string]struct{}{
	"the": {}, "is": {}, "not": {}, "and": {}, "with": {}, "for": {},
	"on": {}, "of": {}, "to": {}, "from": {}, "by": {}, "at": {},
	"or": {}, "but": {}, "if": {}, "then": {}, "when": {}, "how": {},
	"what": {}, "why": {}, "who": {}, "done": {}, "failed": {},
	"finished": {}, "started": {}, "completed": {}, "error": {},
	"warning": {}, "thanks": {}, "you": {}, "your": {},
	"we": {}, "it": {}, "this": {}, "that": {}, "these": {},
	"those": {}, "were": {}, "are": {}, "be": {},
	"been": {}, "has": {}, "have": {}, "had": {}, "can": {},
	"will": {}, "would": {}, "should": {}, "could": {}, "just": {},
	"only": {}, "still": {}, "already": {}, "now": {}, "today": {},
	"tomorrow": {}, "waiting": {}, "running": {}, "please": {},
	"needs": {}, "need": {}, "succeeded": {},
}