From 43efffb01ca78a8165f63d427de6cffa480a40a0 Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Tue, 30 Dec 2025 22:25:59 +0100 Subject: Move gettext --- README.md | 8 +- ki18n/cmd/gettext/gettext.go | 213 ++++++++++++++++++++++++++++++++++++++ ki18n/cmd/gettext/gettext_test.go | 80 ++++++++++++++ ki18n/gettext/gettext.go | 213 -------------------------------------- ki18n/gettext/gettext_test.go | 80 -------------- 5 files changed, 298 insertions(+), 296 deletions(-) create mode 100644 ki18n/cmd/gettext/gettext.go create mode 100644 ki18n/cmd/gettext/gettext_test.go delete mode 100644 ki18n/gettext/gettext.go delete mode 100644 ki18n/gettext/gettext_test.go diff --git a/README.md b/README.md index 690b6ca..74e923a 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,12 @@ - [ ] Ignore regex - [ ] Log level -## gettext +## ki18n -- Add gettext to CI -- Add time handling +- [ ] Document +- [ ] Add gettext to CI +- [ ] Add time handling +- [ ] Use request data for default lang 1. `go run github.com/martinlehoux/kagamigo/cmd/gettext -write` 2. Use `Tr("key", value...)` in you templ files diff --git a/ki18n/cmd/gettext/gettext.go b/ki18n/cmd/gettext/gettext.go new file mode 100644 index 0000000..dbf82d0 --- /dev/null +++ b/ki18n/cmd/gettext/gettext.go @@ -0,0 +1,213 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/martinlehoux/kagamigo/kcore" + "golang.org/x/exp/slog" + "gopkg.in/yaml.v3" +) + +func isIdentifierRune(char rune) bool { + return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '.' +} + +type ArgsParser struct { + argsCount int + currentBlock string + currentNesting int + currentStringLiteral bool +} + +func (p *ArgsParser) parseStringLiteral(char rune) { + switch char { + case '"': + p.currentStringLiteral = false + p.currentBlock = "" + default: + p.currentBlock += string(char) + } +} + +func (p *ArgsParser) parseNonStringLiteral(char rune) { + if isIdentifierRune(char) { + p.currentBlock += string(char) + return + } + switch char { + case ' ': + if p.currentBlock != "" { + if p.currentNesting == 0 { + p.argsCount++ + } + p.currentBlock = "" + } + case '(': + p.currentBlock = "" + p.currentNesting++ + case ')': + p.currentBlock = "" + p.currentNesting-- + p.argsCount++ + case '"': + p.currentStringLiteral = true + } +} + +func (p *ArgsParser) Parse(args string) { + for _, char := range args { + switch p.currentStringLiteral { + case true: + p.parseStringLiteral(char) + case false: + p.parseNonStringLiteral(char) + } + } +} + +func (p ArgsParser) ArgsCount() int { + if p.currentBlock != "" { + p.argsCount++ + } + return p.argsCount +} + +func extractKeys(content string) map[string]int { + extractedKeys := make(map[string]int, 0) + reg := regexp.MustCompile(`\{ ?((?:login\.)?Tr\("[^"]*"[^}]*\) ?)\}`) + matches := reg.FindAllStringSubmatch(content, -1) + for _, match := range matches { + fakePackage := fmt.Sprintf(`package main + + func main() { + %s + } + `, match[1]) + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", fakePackage, 0) + kcore.Expect(err, "error parsing file") + main := f.Decls[0].(*ast.FuncDecl) + call := main.Body.List[0].(*ast.ExprStmt).X.(*ast.CallExpr) + isTrCall := false + switch call.Fun.(type) { + case *ast.SelectorExpr: + selector := call.Fun.(*ast.SelectorExpr) + isTrCall = selector.Sel.Name == "Tr" + + case *ast.Ident: + ident := call.Fun.(*ast.Ident) + isTrCall = ident.Name == "Tr" + } + if isTrCall { + keyLiteral := call.Args[1].(*ast.BasicLit).Value + key, err := strconv.Unquote(keyLiteral) + if err != nil { + slog.Warn("error unquoting key", "key", keyLiteral) + key = strings.Trim(keyLiteral, `"`) + } + extractedKeys[key] = len(call.Args) - 2 + } + } + return extractedKeys +} + +func extractAllKeys() map[string]int { + extractedKeys := make(map[string]int, 0) + + err := filepath.Walk(".", func(path string, info fs.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".templ" { + content, err := os.ReadFile(path) // #nosec G304 + kcore.Expect(err, "error reading file") + for key, value := range extractKeys(string(content)) { + extractedKeys[key] = value + } + } + return nil + }) + kcore.Expect(err, "error walking templates directory") + + return extractedKeys +} + +func getOrCreateLocale(lang string, logger *slog.Logger) map[string]string { + currentLocales := make(map[string]string, 0) + locales, err := os.ReadFile(filepath.Join("locales", lang, "index.yml")) // #nosec G304 + if errors.Is(err, os.ErrNotExist) { + logger.Info("no locales file found, creating") + err = os.MkdirAll(filepath.Join("locales", lang), 0o700) + kcore.Expect(err, "error creating directory") + err = os.WriteFile(filepath.Join("locales", lang, "index.yml"), []byte{}, 0o600) + kcore.Expect(err, "error writing file") + locales, err = os.ReadFile(filepath.Join("locales", lang, "index.yml")) // #nosec G304 + kcore.Expect(err, "error reading file") + } else { + kcore.Expect(err, "error reading file") + } + kcore.Expect(yaml.Unmarshal(locales, ¤tLocales), "error unmarshalling yaml") + return currentLocales +} + +func main() { + write := flag.Bool("write", false, "write new locales") + flag.Parse() + baseLogger := slog.Default() + langs := [...]string{"en-GB", "fr-FR"} + + extractedKeys := extractAllKeys() + baseLogger.Info("extracted keys from templates", slog.Int("count", len(extractedKeys))) + + for _, lang := range langs { + logger := baseLogger.With(slog.String("lang", lang)) + currentLocales := getOrCreateLocale(lang, logger) + newLocales := make(map[string]string, 0) + + correctLocales := 0 + for key, translation := range currentLocales { + currentArgsCount := strings.Count(translation, "%") + expectedArgsCount, ok := extractedKeys[key] + switch { + case !ok: + logger.Info(`found unused key`, slog.String("key", key)) + case currentArgsCount != expectedArgsCount: + logger.Info(`found translation with incorrect number of arguments`, slog.String("key", key), slog.Int("current", currentArgsCount), slog.Int("expected", expectedArgsCount)) + newLocales[key] = "" + case translation == "": + newLocales[key] = "" + default: + newLocales[key] = currentLocales[key] + correctLocales++ + } + } + + for key := range extractedKeys { + if _, ok := currentLocales[key]; !ok { + logger.Info(`found missing key`, slog.String("key", key)) + newLocales[key] = "" + } + } + var completion string + if len(extractedKeys) > 0 { + completion = fmt.Sprintf("%d%%", correctLocales*100/len(extractedKeys)) + } else { + completion = "?%" + } + logger.Info("finished checking locales", slog.Int("count", len(newLocales)), slog.Int("correct", correctLocales), slog.String("completion", completion)) + + if *write { + content, err := yaml.Marshal(newLocales) + kcore.Expect(err, "error marshalling yaml") + kcore.Expect(os.WriteFile(filepath.Join("locales", lang, "index.yml"), content, 0o600), "error writing file") + } + } +} diff --git a/ki18n/cmd/gettext/gettext_test.go b/ki18n/cmd/gettext/gettext_test.go new file mode 100644 index 0000000..14c7160 --- /dev/null +++ b/ki18n/cmd/gettext/gettext_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractKeys(t *testing.T) { + content := `{ login.Tr("fr", "Hello") }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 0, keys["Hello"]) +} + +func TestExtractKeysWithoutSpaces(t *testing.T) { + content := `{login.Tr("fr", "Hello")}` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 0, keys["Hello"]) +} + +func TestExtractKeysWithOneArgs(t *testing.T) { + content := `{ login.Tr("fr", "approveButton", a.Username) }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 1, keys["approveButton"]) +} + +func TestExtractKeysWithSeveralArgs(t *testing.T) { + content := `{ login.Tr("fr", "approveButton", a.Username, a.Age, a.Email) }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 3, keys["approveButton"]) +} + +func TestExtractKeysWithComplexArgs(t *testing.T) { + content := `{ login.Tr("fr", "raceStart_chosen", a.StartAt.Format("Monday, January 2, 2006 at 15:04")) }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 1, keys["raceStart_chosen"]) +} + +func TestExtractMultipleKeys(t *testing.T) { + content := `{ login.Tr("fr", "test_1", a.StartAt.Format("Monday, January 2, 2006 at 15:04")) }{ login.Tr("fr", "test_2", a.Test) }` + + keys := extractKeys(content) + + assert.Len(t, keys, 2) + assert.Equal(t, 1, keys["test_1"]) + assert.Equal(t, 1, keys["test_2"]) +} + +func TestExtractKeyFromSimpleTrFunc(t *testing.T) { + content := `{ Tr("fr", "test") }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 0, keys["test"]) +} + +func TestExtractKeyWithSpan(t *testing.T) { + content := `{ Tr("fr", "Hello %s", username) }` + + keys := extractKeys(content) + + assert.Len(t, keys, 1) + assert.Equal(t, 1, keys["Hello %s"]) +} diff --git a/ki18n/gettext/gettext.go b/ki18n/gettext/gettext.go deleted file mode 100644 index dbf82d0..0000000 --- a/ki18n/gettext/gettext.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - "errors" - "flag" - "fmt" - "go/ast" - "go/parser" - "go/token" - "io/fs" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - - "github.com/martinlehoux/kagamigo/kcore" - "golang.org/x/exp/slog" - "gopkg.in/yaml.v3" -) - -func isIdentifierRune(char rune) bool { - return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char == '.' -} - -type ArgsParser struct { - argsCount int - currentBlock string - currentNesting int - currentStringLiteral bool -} - -func (p *ArgsParser) parseStringLiteral(char rune) { - switch char { - case '"': - p.currentStringLiteral = false - p.currentBlock = "" - default: - p.currentBlock += string(char) - } -} - -func (p *ArgsParser) parseNonStringLiteral(char rune) { - if isIdentifierRune(char) { - p.currentBlock += string(char) - return - } - switch char { - case ' ': - if p.currentBlock != "" { - if p.currentNesting == 0 { - p.argsCount++ - } - p.currentBlock = "" - } - case '(': - p.currentBlock = "" - p.currentNesting++ - case ')': - p.currentBlock = "" - p.currentNesting-- - p.argsCount++ - case '"': - p.currentStringLiteral = true - } -} - -func (p *ArgsParser) Parse(args string) { - for _, char := range args { - switch p.currentStringLiteral { - case true: - p.parseStringLiteral(char) - case false: - p.parseNonStringLiteral(char) - } - } -} - -func (p ArgsParser) ArgsCount() int { - if p.currentBlock != "" { - p.argsCount++ - } - return p.argsCount -} - -func extractKeys(content string) map[string]int { - extractedKeys := make(map[string]int, 0) - reg := regexp.MustCompile(`\{ ?((?:login\.)?Tr\("[^"]*"[^}]*\) ?)\}`) - matches := reg.FindAllStringSubmatch(content, -1) - for _, match := range matches { - fakePackage := fmt.Sprintf(`package main - - func main() { - %s - } - `, match[1]) - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, "", fakePackage, 0) - kcore.Expect(err, "error parsing file") - main := f.Decls[0].(*ast.FuncDecl) - call := main.Body.List[0].(*ast.ExprStmt).X.(*ast.CallExpr) - isTrCall := false - switch call.Fun.(type) { - case *ast.SelectorExpr: - selector := call.Fun.(*ast.SelectorExpr) - isTrCall = selector.Sel.Name == "Tr" - - case *ast.Ident: - ident := call.Fun.(*ast.Ident) - isTrCall = ident.Name == "Tr" - } - if isTrCall { - keyLiteral := call.Args[1].(*ast.BasicLit).Value - key, err := strconv.Unquote(keyLiteral) - if err != nil { - slog.Warn("error unquoting key", "key", keyLiteral) - key = strings.Trim(keyLiteral, `"`) - } - extractedKeys[key] = len(call.Args) - 2 - } - } - return extractedKeys -} - -func extractAllKeys() map[string]int { - extractedKeys := make(map[string]int, 0) - - err := filepath.Walk(".", func(path string, info fs.FileInfo, err error) error { - if !info.IsDir() && filepath.Ext(path) == ".templ" { - content, err := os.ReadFile(path) // #nosec G304 - kcore.Expect(err, "error reading file") - for key, value := range extractKeys(string(content)) { - extractedKeys[key] = value - } - } - return nil - }) - kcore.Expect(err, "error walking templates directory") - - return extractedKeys -} - -func getOrCreateLocale(lang string, logger *slog.Logger) map[string]string { - currentLocales := make(map[string]string, 0) - locales, err := os.ReadFile(filepath.Join("locales", lang, "index.yml")) // #nosec G304 - if errors.Is(err, os.ErrNotExist) { - logger.Info("no locales file found, creating") - err = os.MkdirAll(filepath.Join("locales", lang), 0o700) - kcore.Expect(err, "error creating directory") - err = os.WriteFile(filepath.Join("locales", lang, "index.yml"), []byte{}, 0o600) - kcore.Expect(err, "error writing file") - locales, err = os.ReadFile(filepath.Join("locales", lang, "index.yml")) // #nosec G304 - kcore.Expect(err, "error reading file") - } else { - kcore.Expect(err, "error reading file") - } - kcore.Expect(yaml.Unmarshal(locales, ¤tLocales), "error unmarshalling yaml") - return currentLocales -} - -func main() { - write := flag.Bool("write", false, "write new locales") - flag.Parse() - baseLogger := slog.Default() - langs := [...]string{"en-GB", "fr-FR"} - - extractedKeys := extractAllKeys() - baseLogger.Info("extracted keys from templates", slog.Int("count", len(extractedKeys))) - - for _, lang := range langs { - logger := baseLogger.With(slog.String("lang", lang)) - currentLocales := getOrCreateLocale(lang, logger) - newLocales := make(map[string]string, 0) - - correctLocales := 0 - for key, translation := range currentLocales { - currentArgsCount := strings.Count(translation, "%") - expectedArgsCount, ok := extractedKeys[key] - switch { - case !ok: - logger.Info(`found unused key`, slog.String("key", key)) - case currentArgsCount != expectedArgsCount: - logger.Info(`found translation with incorrect number of arguments`, slog.String("key", key), slog.Int("current", currentArgsCount), slog.Int("expected", expectedArgsCount)) - newLocales[key] = "" - case translation == "": - newLocales[key] = "" - default: - newLocales[key] = currentLocales[key] - correctLocales++ - } - } - - for key := range extractedKeys { - if _, ok := currentLocales[key]; !ok { - logger.Info(`found missing key`, slog.String("key", key)) - newLocales[key] = "" - } - } - var completion string - if len(extractedKeys) > 0 { - completion = fmt.Sprintf("%d%%", correctLocales*100/len(extractedKeys)) - } else { - completion = "?%" - } - logger.Info("finished checking locales", slog.Int("count", len(newLocales)), slog.Int("correct", correctLocales), slog.String("completion", completion)) - - if *write { - content, err := yaml.Marshal(newLocales) - kcore.Expect(err, "error marshalling yaml") - kcore.Expect(os.WriteFile(filepath.Join("locales", lang, "index.yml"), content, 0o600), "error writing file") - } - } -} diff --git a/ki18n/gettext/gettext_test.go b/ki18n/gettext/gettext_test.go deleted file mode 100644 index 14c7160..0000000 --- a/ki18n/gettext/gettext_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestExtractKeys(t *testing.T) { - content := `{ login.Tr("fr", "Hello") }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 0, keys["Hello"]) -} - -func TestExtractKeysWithoutSpaces(t *testing.T) { - content := `{login.Tr("fr", "Hello")}` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 0, keys["Hello"]) -} - -func TestExtractKeysWithOneArgs(t *testing.T) { - content := `{ login.Tr("fr", "approveButton", a.Username) }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 1, keys["approveButton"]) -} - -func TestExtractKeysWithSeveralArgs(t *testing.T) { - content := `{ login.Tr("fr", "approveButton", a.Username, a.Age, a.Email) }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 3, keys["approveButton"]) -} - -func TestExtractKeysWithComplexArgs(t *testing.T) { - content := `{ login.Tr("fr", "raceStart_chosen", a.StartAt.Format("Monday, January 2, 2006 at 15:04")) }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 1, keys["raceStart_chosen"]) -} - -func TestExtractMultipleKeys(t *testing.T) { - content := `{ login.Tr("fr", "test_1", a.StartAt.Format("Monday, January 2, 2006 at 15:04")) }{ login.Tr("fr", "test_2", a.Test) }` - - keys := extractKeys(content) - - assert.Len(t, keys, 2) - assert.Equal(t, 1, keys["test_1"]) - assert.Equal(t, 1, keys["test_2"]) -} - -func TestExtractKeyFromSimpleTrFunc(t *testing.T) { - content := `{ Tr("fr", "test") }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 0, keys["test"]) -} - -func TestExtractKeyWithSpan(t *testing.T) { - content := `{ Tr("fr", "Hello %s", username) }` - - keys := extractKeys(content) - - assert.Len(t, keys, 1) - assert.Equal(t, 1, keys["Hello %s"]) -} -- cgit v1.3