autogits/common/git_utils.go

package common

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"slices"
	"strings"
	"sync"
)

//go:generate mockgen -source=git_utils.go -destination=mock/git_utils.go -typed

type GitSubmoduleLister interface {
	GitSubmoduleList(gitPath, commitId string) (submoduleList map[string]string, err error)
	GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool)
}

type GitDirectoryLister interface {
	GitDirectoryList(gitPath, commitId string) (dirlist map[string]string, err error)
}

type GitSubmoduleFileConflictResolver interface {
	GitResolveConflicts(cwd, MergeBase, Head, MergeHead string) error
	GitResolveSubmoduleFileConflict(s GitStatusData, cwd, mergeBase, head, mergeHead string) error
}

type GitStatusLister interface {
	GitStatus(cwd string) ([]GitStatusData, error)
}

type GitDiffLister interface {
	GitDiff(cwd, base, head string) (string, error)
}

type Git interface {
	// error if git, but wrong remote
	GitClone(repo, branch, remoteUrl string) (string, error) // clone, or check if path is already checked out remote and force pulls, error otherwise. Return remotename, errror

	GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error)
	GitCatFile(cwd, commitId, filename string) (data []byte, err error)
	GetPath() string

	GitBranchHead(gitDir, branchName string) (string, error)
	GitRemoteHead(gitDir, remoteName, branchName string) (string, error)
	io.Closer

	GitSubmoduleLister
	GitDirectoryLister
	GitStatusLister

	GitExecWithOutputOrPanic(cwd string, params ...string) string
	GitExecOrPanic(cwd string, params ...string)
	GitExec(cwd string, params ...string) error
	GitExecWithOutput(cwd string, params ...string) (string, error)
	GitExecQuietOrPanic(cwd string, params ...string)

	GitDiffLister
	GitSubmoduleFileConflictResolver
}

type GitHandlerImpl struct {
	GitPath     string
	GitCommiter string
	GitEmail    string

	lock  *sync.Mutex
	quiet bool
}

func (s *GitHandlerImpl) GetPath() string {
	return s.GitPath
}

type GitHandlerGenerator interface {
	CreateGitHandler(org string) (Git, error)
	ReadExistingPath(org string) (Git, error)

	ReleaseLock(path string)
}

type gitHandlerGeneratorImpl struct {
	path       string
	git_author string
	email      string

	lock_lock sync.Mutex
	lock      map[string]*sync.Mutex // per org
}

func AllocateGitWorkTree(basePath, gitAuthor, email string) (*gitHandlerGeneratorImpl, error) {
	if fi, err := os.Stat(basePath); err != nil || !fi.IsDir() {
		return nil, fmt.Errorf("Git basepath not a valid directory: %s %w", basePath, err)
	}

	if fi, err := os.Stat(basePath); err != nil {
		if os.IsNotExist(err) {
			if err = os.MkdirAll(basePath, 0o700); err != nil {
				return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", basePath, err)
			}
		} else {
			return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", basePath, err)
		}
	} else if !fi.IsDir() {
		return nil, fmt.Errorf("Invalid git directory structure: %s != directory", basePath)
	}

	return &gitHandlerGeneratorImpl{
		path:       basePath,
		git_author: gitAuthor,
		email:      email,

		lock: make(map[string]*sync.Mutex),
	}, nil
}

func (s *gitHandlerGeneratorImpl) CreateGitHandler(org string) (Git, error) {
	path := path.Join(s.path, org)
	if fs, err := os.Stat(path); (err != nil && !os.IsNotExist(err)) || (err == nil && !fs.IsDir()) {
		return nil, err
	} else if err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(path, 0o777); err != nil && !os.IsExist(err) {
			return nil, err
		}
	}

	return s.ReadExistingPath(org)
}

func (s *gitHandlerGeneratorImpl) ReadExistingPath(org string) (Git, error) {
	LogDebug("Locking git org:", org)
	s.lock_lock.Lock()
	defer s.lock_lock.Unlock()

	if _, ok := s.lock[org]; !ok {
		s.lock[org] = &sync.Mutex{}
	}
	s.lock[org].Lock()

	git := &GitHandlerImpl{
		GitCommiter: s.git_author,
		GitEmail:    s.email,
		GitPath:     path.Join(s.path, org),
		lock:        s.lock[org],
	}

	return git, nil
}

func (s *gitHandlerGeneratorImpl) ReleaseLock(org string) {
	m, ok := s.lock[org]
	if ok {
		LogDebug("Unlocking git org:", org)
		m.Unlock()
	}
}

//func (h *GitHandler) ProcessBranchList() []string {
//	if h.HasError() {
//		return make([]string, 0)
//	}
//
//	trackedBranches, err := os.ReadFile(path.Join(h.GitPath, DefaultGitPrj, TrackedBranchesFile))
//	if err != nil {
//		if errors.Is(err, os.ErrNotExist) {
//			trackedBranches = []byte("factory")
//		} else {
//			h.LogError("file error reading '%s' file in repo", TrackedBranchesFile)
//			h.Error = err
//			return make([]string, 0)
//		}
//	}
//
//	return strings.Split(string(trackedBranches), "\n")
//}

type GitReference struct {
	Branch string
	Id     string
}

type GitReferences struct {
	refs []GitReference
}

func (refs *GitReferences) addReference(id, branch string) {
	for _, ref := range refs.refs {
		if ref.Id == id && ref.Branch == branch {
			return
		}
	}

	refs.refs = append(refs.refs, GitReference{Branch: branch, Id: id})
}

func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error) {
	LogDebug("Cloning", remoteUrl, "  repo:", repo, "  branch:", branch)
	remoteUrlComp, err := ParseGitRemoteUrl(remoteUrl)
	if err != nil {
		return "", fmt.Errorf("Cannot parse remote URL: %w", err)
	}
	remoteBranch := "HEAD"
	if len(branch) == 0 && remoteUrlComp != nil && remoteUrlComp.Commit != "HEAD" {
		branch = remoteUrlComp.Commit
		remoteBranch = branch
	} else if len(branch) > 0 {
		remoteBranch = branch
	}
	remoteName := remoteUrlComp.RemoteName()
	if remoteUrlComp != nil {
		LogDebug("Clone", *remoteUrlComp, " -> ", remoteName)
	} else {
		LogDebug("Clone", "[default]  -> ", remoteName)
	}

	remoteRef := remoteName + "/" + remoteBranch
	if fi, err := os.Stat(path.Join(e.GitPath, repo)); os.IsNotExist(err) {
		if err = e.GitExec("", "clone", "--origin", remoteName, remoteUrl, repo); err != nil {
			return remoteName, err
		}
	} else if err != nil || !fi.IsDir() {
		return remoteName, fmt.Errorf("Clone location not a directory or Stat error: %w", err)
	} else {
		if u, err := e.GitExecWithOutput(repo, "remote", "get-url", remoteName); err != nil {
			e.GitExecOrPanic(repo, "remote", "add", remoteName, remoteUrl)
		} else if clonedRemote := strings.TrimSpace(u); clonedRemote != remoteUrl {
			e.GitExecOrPanic(repo, "remote", "set-url", remoteName, remoteUrl)
		}

		// check if we have submodule to deinit
		if list, _ := e.GitSubmoduleList(repo, "HEAD"); len(list) > 0 {
			e.GitExecQuietOrPanic(repo, "submodule", "deinit", "--all", "--force")
		}

		e.GitExecOrPanic(repo, "fetch", "--prune", remoteName, remoteBranch)
	}
	/*
		refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
		if err != nil {
			LogError("Cannot read HEAD of remote", remoteName)
			return remoteName, fmt.Errorf("Cannot read HEAD of remote %s", remoteName)
		}

		refs := string(refsBytes)
		if refs[0:5] != "ref: " {
			LogError("Unexpected format of remote HEAD ref:", refs)
			return remoteName, fmt.Errorf("Unexpected format of remote HEAD ref: %s", refs)
		}

		if len(branch) == 0 || branch == "HEAD" {
			remoteRef = strings.TrimSpace(refs[5:])
			branch = remoteRef[strings.LastIndex(remoteRef, "/")+1:]
			LogDebug("remoteRef", remoteRef)
			LogDebug("branch", branch)
		}
	*/
	args := []string{"fetch", "--prune", remoteName, branch}
	if strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "rev-parse", "--is-shallow-repository")) == "true" {
		args = slices.Insert(args, 1, "--unshallow")
	}
	e.GitExecOrPanic(repo, args...)
	return remoteName, e.GitExec(repo, "checkout", "-f", "--track", "-B", branch, remoteRef)
}

func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--heads", "--hash", branchName)
	if err != nil {
		return "", fmt.Errorf("Can't find default branch: %s", branchName)
	}

	id = strings.TrimSpace(SplitLines(id)[0])
	if len(id) < 10 {
		return "", fmt.Errorf("Can't find branch: %s", branchName)
	}

	return id, nil
}

func (e *GitHandlerImpl) GitRemoteHead(gitDir, remote, branchName string) (string, error) {
	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/remotes/"+remote+"/"+branchName)
	if err != nil {
		return "", fmt.Errorf("Can't find default branch: %s", branchName)
	}

	return strings.TrimSpace(id), nil
}

func (e *GitHandlerImpl) Close() error {
	LogDebug("Unlocking git lock")
	e.lock.Unlock()
	return nil
}

type writeFunc func(data []byte) (int, error)

func (f writeFunc) Write(data []byte) (int, error) {
	return f(data)
}

func (h writeFunc) UnmarshalText(text []byte) error {
	_, err := h.Write(text)
	return err
}

func (h writeFunc) Close() error {
	_, err := h.Write(nil)
	return err
}

func (e *GitHandlerImpl) GitExecWithOutputOrPanic(cwd string, params ...string) string {
	out, err := e.GitExecWithOutput(cwd, params...)
	if err != nil {
		LogError("git command failed:", params, "@", cwd, "err:", err)
		panic(err)
	}
	return out
}

func (e *GitHandlerImpl) GitExecOrPanic(cwd string, params ...string) {
	if err := e.GitExec(cwd, params...); err != nil {
		LogError("git command failed:", params, "@", cwd, "err:", err)
		panic(err)
	}
}

func (e *GitHandlerImpl) GitExec(cwd string, params ...string) error {
	_, err := e.GitExecWithOutput(cwd, params...)
	return err
}

var ExtraGitParams []string

func (e *GitHandlerImpl) GitExecWithOutput(cwd string, params ...string) (string, error) {
	cmd := exec.Command("/usr/bin/git", params...)
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_CONFIG_GLOBAL=/dev/null",
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_LFS_SKIP_PUSH=1",
		"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
	}
	if len(e.GitEmail) > 0 {
		cmd.Env = append(cmd.Env, "EMAIL="+e.GitEmail)
	}
	if len(e.GitCommiter) > 0 {
		cmd.Env = append(cmd.Env,
			"GIT_AUTHOR_NAME="+e.GitCommiter,
			"GIT_COMMITTER_NAME="+e.GitCommiter,
		)
	}

	if len(ExtraGitParams) > 0 {
		cmd.Env = append(cmd.Env, ExtraGitParams...)
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stdin = nil

	LogDebug("git execute @", cwd, ":", cmd.Args)
	out, err := cmd.CombinedOutput()
	if !e.quiet {
		LogDebug(string(out))
	}
	if err != nil {
		LogError("git", cmd.Args, " error:", err)
		return "", fmt.Errorf("error executing: git %#v \n%s\n err: %w", cmd.Args, out, err)
	}

	return string(out), nil
}

func (e *GitHandlerImpl) GitExecQuietOrPanic(cwd string, params ...string) {
	e.quiet = true
	e.GitExecOrPanic(cwd, params...)
	e.quiet = false
	return
}

type ChanIO struct {
	ch chan byte
}

func (c *ChanIO) Write(p []byte) (int, error) {
	for _, b := range p {
		c.ch <- b
	}
	return len(p), nil
}

// read at least 1 byte, but don't block if nothing more in channel
func (c *ChanIO) Read(data []byte) (idx int, err error) {
	var ok bool

	data[idx], ok = <-c.ch
	if !ok {
		err = io.EOF
		return
	}
	idx++

	for len(c.ch) > 0 && idx < len(data) {
		data[idx], ok = <-c.ch
		if !ok {
			err = io.EOF
			return
		}

		idx++
	}

	return
}

type GitMsg struct {
	hash     string
	itemType string
	size     int
}

type GitCommit struct {
	Tree string
	Msg  string
}

type GitTreeEntry struct {
	name string
	mode int
	hash string

	size int
}

type GitTree struct {
	items []GitTreeEntry
}

func (t *GitTreeEntry) isSubmodule() bool {
	return (t.mode & 0170000) == 0160000
}

func (t *GitTreeEntry) isTree() bool {
	return (t.mode & 0170000) == 0040000
}

func (t *GitTreeEntry) isBlob() bool {
	return !t.isTree() && !t.isSubmodule()
}

func parseGitMsg(data <-chan byte) (GitMsg, error) {
	var id []byte = make([]byte, 64)
	var msgType []byte = make([]byte, 16)
	var size int

	pos := 0
	for c := <-data; c != ' '; c = <-data {
		if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') {
			id[pos] = c
			pos++
		} else {
			return GitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos)
		}
	}
	id = id[:pos]

	pos = 0
	var c byte
	for c = <-data; c != ' ' && c != '\x00'; c = <-data {
		if c >= 'a' && c <= 'z' {
			msgType[pos] = c
			pos++
		} else {
			return GitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos)
		}
	}
	msgType = msgType[:pos]

	switch string(msgType) {
	case "commit", "tree", "blob":
		break
	case "missing":
		if c != '\x00' {
			return GitMsg{}, fmt.Errorf("Missing format weird")
		}
		return GitMsg{
			hash:     string(id[:]),
			itemType: "missing",
			size:     0,
		}, fmt.Errorf("Object not found: '%s'", string(id))
	default:
		return GitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
	}

	for c = <-data; c != '\000'; c = <-data {
		if c >= '0' && c <= '9' {
			size = size*10 + (int(c) - '0')
		} else {
			return GitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c)
		}
	}

	return GitMsg{
		hash:     string(id[:]),
		itemType: string(msgType),
		size:     size,
	}, nil
}

func parseGitCommitHdr(oldHdr [2]string, data <-chan byte) ([2]string, int, error) {
	hdr := make([]byte, 0, 60)
	val := make([]byte, 0, 1000)

	c := <-data
	size := 1
	if c != '\n' { // end of header marker
		for ; c != ' '; c = <-data {
			hdr = append(hdr, c)
			size++
		}
		if size == 1 { // continuation header here
			hdr = []byte(oldHdr[0])
			val = append([]byte(oldHdr[1]), '\n')
		}
		for c := <-data; c != '\n'; c = <-data {
			val = append(val, c)
			size++
		}
		size++
	}

	return [2]string{string(hdr), string(val)}, size, nil
}

func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
	msg := make([]byte, 0, l)

	for c := <-data; c != '\x00'; c = <-data {
		msg = append(msg, c)
		l--
	}

	if l != 0 {
		return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l)
	}

	return string(msg), nil
}

func parseGitCommit(data <-chan byte) (GitCommit, error) {
	hdr, err := parseGitMsg(data)
	if err != nil {
		return GitCommit{}, err
	} else if hdr.itemType != "commit" {
		return GitCommit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType)
	}

	var c GitCommit
	l := hdr.size
	for {
		var hdr [2]string
		hdr, size, err := parseGitCommitHdr(hdr, data)
		if err != nil {
			return GitCommit{}, nil
		}
		l -= size

		if size == 1 {
			break
		}

		switch hdr[0] {
		case "tree":
			c.Tree = hdr[1]
		}
	}

	c.Msg, err = parseGitCommitMsg(data, l)
	return c, err
}

func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
	var e GitTreeEntry

	for c := <-data; c != ' '; c = <-data {
		e.mode = e.mode*8 + int(c-'0')
		e.size++
	}
	e.size++

	name := make([]byte, 0, 128)
	for c := <-data; c != '\x00'; c = <-data {
		name = append(name, c)
		e.size++
	}
	e.size++
	e.name = string(name)

	const hexBinToAscii = "0123456789abcdef"

	hash := make([]byte, 0, hashLen*2)
	for range hashLen {
		c := <-data
		hash = append(hash, hexBinToAscii[((c&0xF0)>>4)], hexBinToAscii[c&0xF])
	}
	e.hash = string(hash)
	e.size += hashLen

	return e, nil
}

func parseGitTree(data <-chan byte) (GitTree, error) {
	hdr, err := parseGitMsg(data)
	if err != nil {
		return GitTree{}, err
	}

	// max capacity to length of hash
	t := GitTree{items: make([]GitTreeEntry, 0, hdr.size/len(hdr.hash))}
	parsedLen := 0
	for parsedLen < hdr.size {
		entry, err := parseTreeEntry(data, len(hdr.hash)/2)
		if err != nil {
			return GitTree{}, nil
		}

		t.items = append(t.items, entry)
		parsedLen += entry.size
	}
	c := <-data // \0 read

	if c != '\x00' {
		return t, fmt.Errorf("Unexpected character during git tree data read")
	}

	if parsedLen != hdr.size {
		return t, fmt.Errorf("Invalid size of git tree data")
	}

	return t, nil
}

func parseGitBlob(data <-chan byte) ([]byte, error) {
	hdr, err := parseGitMsg(data)
	if err != nil {
		return []byte{}, err
	}

	d := make([]byte, hdr.size)
	for l := 0; l < hdr.size; l++ {
		d[l] = <-data
	}
	eob := <-data
	if eob != '\x00' {
		return d, fmt.Errorf("invalid byte read in parseGitBlob")
	}

	return d, nil
}

func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) {
	var done sync.Mutex

	done.Lock()
	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
	parsedCommits = make([]GitCommit, 0, len(commitIDs))

	go func() {
		defer done.Unlock()
		defer close(data_out.ch)

		for _, id := range commitIDs {
			data_out.Write([]byte(id))
			data_out.ch <- '\x00'
			c, e := parseGitCommit(data_in.ch)
			if e != nil {
				err = fmt.Errorf("Error parsing git commit: %w", e)
				return
			}

			parsedCommits = append(parsedCommits, c)
		}
	}()

	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stdout = &data_in
	cmd.Stdin = &data_out
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	if e := cmd.Run(); e != nil {
		LogError(e)
		close(data_in.ch)
		close(data_out.ch)
		return nil, e
	}

	done.Lock()
	return
}

// TODO: support sub-trees
func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte, err error) {
	var done sync.Mutex

	done.Lock()
	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}

	go func() {
		defer done.Unlock()
		defer close(data_out.ch)

		data_out.Write([]byte(commitId))
		data_out.ch <- '\x00'

		var c GitCommit
		c, err = parseGitCommit(data_in.ch)
		if err != nil {
			LogError("Error parsing git commit:", err)
			return
		}
		data_out.Write([]byte(c.Tree))
		data_out.ch <- '\x00'

		var tree GitTree
		tree, err = parseGitTree(data_in.ch)

		if err != nil {
			LogError("Error parsing git tree:", err)
			return
		}

		for _, te := range tree.items {
			if te.isBlob() && te.name == filename {
				LogInfo("blob", te.hash)
				data_out.Write([]byte(te.hash))
				data_out.ch <- '\x00'
				data, err = parseGitBlob(data_in.ch)
				return
			}
		}

		LogError("file not found:", filename)
	}()

	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stdout = &data_in
	cmd.Stdin = &data_out
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	if e := cmd.Run(); e != nil {
		LogError(e)
		close(data_in.ch)
		close(data_out.ch)
		return nil, e
	}
	done.Lock()
	return
}

// return (filename) -> (hash) map for all submodules
func (e *GitHandlerImpl) GitDirectoryList(gitPath, commitId string) (directoryList map[string]string, err error) {
	var done sync.Mutex
	directoryList = make(map[string]string)

	done.Lock()
	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}

	LogDebug("Getting directory for:", commitId)

	go func() {
		defer done.Unlock()
		defer close(data_out.ch)

		data_out.Write([]byte(commitId))
		data_out.ch <- '\x00'
		var c GitCommit
		c, err = parseGitCommit(data_in.ch)
		if err != nil {
			err = fmt.Errorf("Error parsing git commit. Err: %w", err)
			return
		}

		trees := make(map[string]string)
		trees[""] = c.Tree

		for len(trees) > 0 {
			for p, tree := range trees {
				delete(trees, p)

				data_out.Write([]byte(tree))
				data_out.ch <- '\x00'
				var tree GitTree
				tree, err = parseGitTree(data_in.ch)

				if err != nil {
					err = fmt.Errorf("Error parsing git tree: %w", err)
					return
				}

				for _, te := range tree.items {
					if te.isTree() {
						directoryList[p+te.name] = te.hash
					}
				}
			}
		}
	}()

	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, gitPath)
	cmd.Stdout = &data_in
	cmd.Stdin = &data_out
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	if e := cmd.Run(); e != nil {
		LogError(e)
		close(data_in.ch)
		close(data_out.ch)
		return directoryList, e
	}

	done.Lock()
	return directoryList, err
}

// return (filename) -> (hash) map for all submodules
func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleList map[string]string, err error) {
	var done sync.Mutex
	submoduleList = make(map[string]string)

	done.Lock()
	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}

	LogDebug("Getting submodules for:", commitId)

	go func() {
		defer done.Unlock()
		defer close(data_out.ch)

		data_out.Write([]byte(commitId))
		data_out.ch <- '\x00'
		var c GitCommit
		c, err = parseGitCommit(data_in.ch)
		if err != nil {
			err = fmt.Errorf("Error parsing git commit. Err: %w", err)
			return
		}

		trees := make(map[string]string)
		trees[""] = c.Tree

		for len(trees) > 0 {
			for p, tree := range trees {
				delete(trees, p)

				data_out.Write([]byte(tree))
				data_out.ch <- '\x00'
				var tree GitTree
				tree, err = parseGitTree(data_in.ch)

				if err != nil {
					err = fmt.Errorf("Error parsing git tree: %w", err)
					return
				}

				for _, te := range tree.items {
					if te.isTree() {
						trees[p+te.name+"/"] = te.hash
					} else if te.isSubmodule() {
						submoduleList[p+te.name] = te.hash
					}

				}
			}
		}
	}()

	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, gitPath)
	cmd.Stdout = &data_in
	cmd.Stdin = &data_out
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	if e := cmd.Run(); e != nil {
		LogError(e)
		close(data_in.ch)
		close(data_out.ch)
		return submoduleList, e
	}

	done.Lock()
	return submoduleList, err
}

func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool) {
	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
	var wg sync.WaitGroup

	wg.Add(1)
	LogDebug("getting commit id", commitId, "from git at", cwd, "with packageName:", packageName)

	go func() {
		defer func() {
			if recover() != nil {
				subCommitId = ""
				commitId = "ok"
				valid = false
			}
		}()

		defer wg.Done()
		defer close(data_out.ch)

		data_out.Write([]byte(commitId))
		data_out.ch <- '\x00'
		c, err := parseGitCommit(data_in.ch)
		if err != nil {
			LogError("Error parsing git commit:", err)
			panic(err)
		}
		data_out.Write([]byte(c.Tree))
		data_out.ch <- '\x00'
		tree, err := parseGitTree(data_in.ch)

		if err != nil {
			LogError("Error parsing git tree:", err)
			panic(err)
		}

		for _, te := range tree.items {
			if te.name == packageName && te.isSubmodule() {
				subCommitId = te.hash
				return
			}
		}
	}()

	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stdout = &data_in
	cmd.Stdin = &data_out
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	if e := cmd.Run(); e != nil {
		LogError(e)
		close(data_in.ch)
		close(data_out.ch)
		return subCommitId, false
	}

	wg.Wait()
	return subCommitId, len(subCommitId) > 0
}

func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.ByteReader) (Data, error), gitcmd string, args ...string) (Data, error) {
	LogDebug("getting", gitcmd)
	args = append([]string{gitcmd}, args...)
	cmd := exec.Command("/usr/bin/git", args...)
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	out, err := cmd.Output()
	if err != nil {
		LogError("Error running command", cmd.Args, err)
	}

	return dataprocessor(bytes.NewReader(out))
}

func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
	data, err := e.GitExecWithDataParse(cwd, parseGitStatusData, "status", "--porcelain=2", "-z")
	return data.([]GitStatusData), err
}

func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
	LogDebug("getting diff from", base, "..", head)

	cmd := exec.Command("/usr/bin/git", "diff", base+".."+head)
	cmd.Env = []string{
		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
		"GIT_LFS_SKIP_SMUDGE=1",
		"GIT_CONFIG_GLOBAL=/dev/null",
	}
	cmd.Dir = filepath.Join(e.GitPath, cwd)
	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
		LogError(string(data))
		return len(data), nil
	})
	LogDebug("command run:", cmd.Args)
	out, err := cmd.Output()
	if err != nil {
		LogError("Error running command", cmd.Args, err)
	}

	return string(out), nil
}

func (e *GitHandlerImpl) GitDiffIndex(cwd, commit string) ([]GitDiffRawData, error) {
	data, err := e.GitExecWithDataParse("diff-index", parseGitDiffIndexRawData, cwd, "diff-index", "-z", "--raw", "--full-index", "--submodule=short", "HEAD")
	return data.([]GitDiffRawData), err
}

func (git *GitHandlerImpl) GitResolveConflicts(cwd, mergeBase, head, mergeHead string) error {
	status, err := git.GitStatus(cwd)
	if err != nil {
		return fmt.Errorf("Status failed: %w", err)
	}

	// we can only resolve conflicts with .gitmodules
	for _, s := range status {
		if s.Status == GitStatus_Unmerged && s.Path == ".gitmodules" {
			if err := git.GitResolveSubmoduleFileConflict(s, cwd, mergeBase, head, mergeHead); err != nil {
				return err
			}
		} else if s.Status == GitStatus_Unmerged {
			return fmt.Errorf("Cannot automatically resolve conflict: %s", s.Path)
		}
	}

	return git.GitExec(cwd, "-c", "core.editor=true", "merge", "--continue")
}

func (git *GitHandlerImpl) GitResolveSubmoduleFileConflict(s GitStatusData, cwd, mergeBase, head, mergeHead string) error {
	submodules1, err := git.GitSubmoduleList(cwd, mergeBase)
	if err != nil {
		return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
	}
	/*
		submodules2, err := git.GitSubmoduleList(cwd, head)
		if err != nil {
			return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
		}
	*/
	submodules3, err := git.GitSubmoduleList(cwd, mergeHead)
	if err != nil {
		return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
	}

	// find modified submodules in the mergeHead
	modifiedSubmodules := make([]string, 0, 10)
	removedSubmodules := make([]string, 0, 10)
	addedSubmodules := make([]string, 0, 10)

	for submodulePath, oldHash := range submodules1 {
		if newHash, found := submodules3[submodulePath]; found && newHash != oldHash {
			modifiedSubmodules = append(modifiedSubmodules, submodulePath)
		} else if !found {
			removedSubmodules = append(removedSubmodules, submodulePath)
		}
	}
	for submodulePath, _ := range submodules3 {
		if _, found := submodules1[submodulePath]; !found {
			addedSubmodules = append(addedSubmodules, submodulePath)
		}
	}

	// We need to adjust the `submodules` list by the pending changes to the index
	s1, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[0])
	if err != nil {
		return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
	}
	s2, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[1])
	if err != nil {
		return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
	}
	s3, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[2])
	if err != nil {
		return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
	}

	_, err = ParseSubmodulesFile(strings.NewReader(s1))
	if err != nil {
		return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
	}
	subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
	if err != nil {
		return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
	}
	subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
	if err != nil {
		return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
	}

	overrideSet := make([]Submodule, 0, len(addedSubmodules)+len(modifiedSubmodules))
	for i := range subs3 {
		if slices.Contains(addedSubmodules, subs3[i].Path) || slices.Contains(modifiedSubmodules, subs3[i].Path) {
			overrideSet = append(overrideSet, subs3[i])
		}
	}
	// merge from subs1 (merge-base), subs2 (changes in base since merge-base HEAD), subs3 (merge_source MERGE_HEAD)
	// this will update submodules
	SubmoduleCompare := func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }
	CompactCompare := func(a, b Submodule) bool { return a.Path == b.Path }

	// remove submodules that are removed in the PR
	subs2 = slices.DeleteFunc(subs2, func(a Submodule) bool { return slices.Contains(removedSubmodules, a.Path) })

	mergedSubs := slices.Concat(overrideSet, subs2)
	slices.SortStableFunc(mergedSubs, SubmoduleCompare)
	filteredSubs := slices.CompactFunc(mergedSubs, CompactCompare)

	out, err := os.Create(path.Join(git.GetPath(), cwd, ".gitmodules"))
	if err != nil {
		return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
	}
	if err = WriteSubmodules(filteredSubs, out); err != nil {
		return fmt.Errorf("Can't write .gitmodules: %w", err)
	}
	if out.Close(); err != nil {
		return fmt.Errorf("Can't close .gitmodules: %w", err)
	}

	git.GitExecOrPanic(cwd, "add", ".gitmodules")
	return nil
}