From d174bb92ed033ac4235c5e7b30bf617575779fb9 Mon Sep 17 00:00:00 2001 From: Martin Kagamino Lehoux Date: Fri, 4 Oct 2024 10:25:53 +0200 Subject: refactor(kdev): Support several record extractors --- cmd/kdev/main.go | 89 ++++++++++++++++++++++++----------------- cmd/kdev/record_extractor.go | 95 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 36 deletions(-) create mode 100644 cmd/kdev/record_extractor.go (limited to 'cmd') diff --git a/cmd/kdev/main.go b/cmd/kdev/main.go index 9efdee2..03ba9b7 100644 --- a/cmd/kdev/main.go +++ b/cmd/kdev/main.go @@ -5,15 +5,19 @@ import ( "fmt" "io/fs" "os" - "os/exec" + "path" "path/filepath" + "runtime/pprof" "slices" - "strconv" "strings" "time" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/martinlehoux/kagamigo/kcore" "github.com/samber/lo" + "github.com/schollz/progressbar/v3" + "golang.org/x/exp/slog" ) type Record struct { @@ -23,25 +27,37 @@ type Record struct { date time.Time } +var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") +var sortBy = flag.String("sort", "date", "Sort by date or random") +var maxRecords = flag.Int("max", 25, "Maximum number of records to display") +var repoPath = flag.String("repo", ".", "Path to the repository") +var afterS = flag.String("after", "2022-09-01", "Only show records after this date") + func main() { var err error - sortBy := flag.String("sort", "date", "Sort by date or random") - maxRecords := flag.Int("max", 25, "Maximum number of records to display") - repoPath := flag.String("repo", ".", "Path to the repository") - afterS := flag.String("after", "2022-09-01", "Only show records after this date") flag.Parse() after, err := time.Parse(time.DateOnly, *afterS) kcore.Expect(err, "Error parsing date") - excludes := []string{".venv", ".git", ".ruff_cache", "db_dumps", ".mypy_cache"} + excludes := []string{".venv", ".git", ".ruff_cache", "db_dumps", ".mypy_cache", "uploads", "__pycache__", ".coverage"} keywords := []string{"# TODO"} records := []Record{} - // repo, err := git.PlainOpen(root) + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + kcore.Expect(err, "Error creating CPU profile") + kcore.Expect(pprof.StartCPUProfile(f), "Error starting CPU profile") + defer pprof.StopCPUProfile() + } + slog.Info("Scanning repository", "repo", *repoPath, "sort", *sortBy, "max", *maxRecords) + repo, err := git.PlainOpen(path.Join(*repoPath, ".git")) + // repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{URL: path.Join(*repoPath, ".git")}) kcore.Expect(err, "Error opening repository") - // ref, err := repo.Head() + ref, err := repo.Head() kcore.Expect(err, "Error getting HEAD") - // commit, err := repo.CommitObject(ref.Hash()) + head, err := repo.CommitObject(ref.Hash()) kcore.Expect(err, "Error getting commit object") + progress := progressbar.Default(-1, "Scanning") err = filepath.WalkDir(*repoPath, func(path string, d fs.DirEntry, err error) error { + relativePath := strings.TrimPrefix(path, *repoPath+"/") if err != nil { return err } @@ -52,7 +68,8 @@ func main() { return nil } if !d.IsDir() { - return processFile(path, keywords, &records, *repoPath) + kcore.Expect(progress.Add(1), "Error incrementing progress") + return processFile(*repoPath, relativePath, keywords, &records, head) } return nil }) @@ -63,46 +80,46 @@ func main() { } else { slices.SortFunc(records, func(a, b Record) int { return -int(a.date.Sub(b.date).Nanoseconds()) }) } + fmt.Println("") for _, record := range records[:*maxRecords] { fmt.Printf("%s\t %s:%d\n", record.date.Format("2006-01-02"), record.path, record.line) } } -func processFile(path string, keywords []string, records *[]Record, repoPath string) error { - content, err := os.ReadFile(path) // #nosec G304 +type MatchingLine struct { + line int + keyword string +} + +func processFile(repoPath string, relativePath string, keywords []string, records *[]Record, head *object.Commit) error { + absolutePath := path.Join(repoPath, relativePath) + content, err := os.ReadFile(absolutePath) // #nosec G304 if err != nil { return err } + lines := strings.Split(string(content), "\n") - for i, line := range lines { + matchingLines := lo.FilterMap(lines, func(line string, i int) (MatchingLine, bool) { for _, keyword := range keywords { if strings.Contains(line, keyword) { - *records = append(*records, extractRecord(keyword, path, i+1, repoPath)) + return MatchingLine{i + 1, keyword}, true } } + return MatchingLine{}, false + }) + if len(matchingLines) == 0 { + return nil } - return nil -} -func extractRecord(keyword string, path string, line int, repoPath string) Record { - record := Record{ - keyword: keyword, - path: path, - line: line, - } - cmd := exec.Command("git", "blame", "-L", fmt.Sprintf("%d,%d", record.line, record.line), "--porcelain", path) // #nosec G204 - cmd.Dir = repoPath - output, err := cmd.Output() - if err != nil { - panic(kcore.Wrap(err, "Error running git blame")) + recordExtractor := NewStatRecordExtractor(absolutePath) + // recordExtractor = NewGoGitRecordExtractor(head, relativePath) + // recordExtractor := &GitRecordExtractor{repoPath: repoPath} + if recordExtractor == nil { + return nil } - for _, blameLine := range strings.Split(string(output), "\n") { - if strings.HasPrefix(blameLine, "author-time ") { - timestamp, err := strconv.ParseInt(strings.TrimPrefix(blameLine, "author-time "), 10, 64) - kcore.Expect(err, "Error parsing timestamp") - record.date = time.Unix(timestamp, 0) - break - } + for _, line := range matchingLines { + *records = append(*records, recordExtractor.Extract(line.keyword, relativePath, line.line)) } - return record + + return nil } diff --git a/cmd/kdev/record_extractor.go b/cmd/kdev/record_extractor.go new file mode 100644 index 0000000..2dc079e --- /dev/null +++ b/cmd/kdev/record_extractor.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "io/fs" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/martinlehoux/kagamigo/kcore" +) + +type RecordExtractor interface { + Extract(keyword string, path string, line int) Record +} + +type StatRecordExtractor struct { + fileInfo fs.FileInfo +} + +func NewStatRecordExtractor(absolutePath string) *StatRecordExtractor { + fileInfo, err := os.Stat(absolutePath) + kcore.Expect(err, "Error getting file info") + + return &StatRecordExtractor{ + fileInfo: fileInfo, + } +} + +func (re *StatRecordExtractor) Extract(keyword string, path string, line int) Record { + return Record{ + keyword: keyword, + path: path, + line: line, + date: re.fileInfo.ModTime(), + } +} + +type GoGitRecordExtractor struct { + blame *git.BlameResult +} + +func NewGoGitRecordExtractor(head *object.Commit, path string) *GoGitRecordExtractor { + blame, err := git.Blame(head, path) + if err == object.ErrFileNotFound { + return nil + } + return &GoGitRecordExtractor{ + blame: blame, + } +} + +func (re *GoGitRecordExtractor) Extract(keyword string, path string, line int) Record { + return Record{ + keyword: keyword, + path: path, + line: line, + date: re.blame.Lines[line-1].Date, + } +} + +type GitRecordExtractor struct { + repoPath string +} + +func (re *GitRecordExtractor) Extract(keyword string, path string, line int) Record { + record := Record{ + keyword: keyword, + path: path, + line: line, + } + gitArgs := []string{"blame", "-L", fmt.Sprintf("%d,%d", line, line), "--porcelain", path} + cmd := exec.Command("git", gitArgs...) // #nosec G204 + cmd.Dir = re.repoPath + output, err := cmd.Output() + + if err != nil { + panic(kcore.Wrap(err, fmt.Sprintf("Error running: git %s", strings.Join(gitArgs, " ")))) + } + + for _, blameLine := range strings.Split(string(output), "\n") { + if strings.HasPrefix(blameLine, "author-time ") { + timestamp, err := strconv.ParseInt(strings.TrimPrefix(blameLine, "author-time "), 10, 64) + kcore.Expect(err, "Error parsing timestamp") + record.date = time.Unix(timestamp, 0) + return record + } + } + kcore.Assert(false, "No date found in git blame output") + return Record{} +} -- cgit v1.3