staging: assume changed directories are packages

Ignore any non-top level direcotries here. This should be fixed
to handle _manifest files

common/git_utils.go

@@ -40,6 +40,10 @@ type GitSubmoduleLister interface {
 	GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool)
 }
 
+type GitDirectoryLister interface {
+	GitDirectoryList(gitPath, commitId string) (dirlist map[string]string, err error)
+}
+
 type GitStatusLister interface {
 	GitStatus(cwd string) ([]GitStatusData, error)
 }
@@ -61,6 +65,7 @@ type Git interface {
 	io.Closer
 
 	GitSubmoduleLister
+	GitDirectoryLister
 	GitStatusLister
 
 	GitExecWithOutputOrPanic(cwd string, params ...string) string
@@ -247,26 +252,26 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
 
 		e.GitExecOrPanic(repo, "fetch", "--prune", remoteName, remoteBranch)
 	}
-/*
-	refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
-	if err != nil {
-		LogError("Cannot read HEAD of remote", remoteName)
-		return remoteName, fmt.Errorf("Cannot read HEAD of remote %s", remoteName)
-	}
+	/*
+		refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
+		if err != nil {
+			LogError("Cannot read HEAD of remote", remoteName)
+			return remoteName, fmt.Errorf("Cannot read HEAD of remote %s", remoteName)
+		}
 
-	refs := string(refsBytes)
-	if refs[0:5] != "ref: " {
-		LogError("Unexpected format of remote HEAD ref:", refs)
-		return remoteName, fmt.Errorf("Unexpected format of remote HEAD ref: %s", refs)
-	}
+		refs := string(refsBytes)
+		if refs[0:5] != "ref: " {
+			LogError("Unexpected format of remote HEAD ref:", refs)
+			return remoteName, fmt.Errorf("Unexpected format of remote HEAD ref: %s", refs)
+		}
 
-	if len(branch) == 0 || branch == "HEAD" {
-		remoteRef = strings.TrimSpace(refs[5:])
-		branch = remoteRef[strings.LastIndex(remoteRef, "/")+1:]
-		LogDebug("remoteRef", remoteRef)
-		LogDebug("branch", branch)
-	}
-*/
+		if len(branch) == 0 || branch == "HEAD" {
+			remoteRef = strings.TrimSpace(refs[5:])
+			branch = remoteRef[strings.LastIndex(remoteRef, "/")+1:]
+			LogDebug("remoteRef", remoteRef)
+			LogDebug("branch", branch)
+		}
+	*/
 	args := []string{"fetch", "--prune", remoteName, branch}
 	if strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "rev-parse", "--is-shallow-repository")) == "true" {
 		args = slices.Insert(args, 1, "--unshallow")
@@ -778,6 +783,80 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
 	return
 }
 
+// return (filename) -> (hash) map for all submodules
+func (e *GitHandlerImpl) GitDirectoryList(gitPath, commitId string) (directoryList map[string]string, err error) {
+	var done sync.Mutex
+	directoryList = make(map[string]string)
+
+	done.Lock()
+	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+
+	LogDebug("Getting directory for:", commitId)
+
+	go func() {
+		defer done.Unlock()
+		defer close(data_out.ch)
+
+		data_out.Write([]byte(commitId))
+		data_out.ch <- '\x00'
+		var c GitCommit
+		c, err = parseGitCommit(data_in.ch)
+		if err != nil {
+			err = fmt.Errorf("Error parsing git commit. Err: %w", err)
+			return
+		}
+
+		trees := make(map[string]string)
+		trees[""] = c.Tree
+
+		for len(trees) > 0 {
+			for p, tree := range trees {
+				delete(trees, p)
+
+				data_out.Write([]byte(tree))
+				data_out.ch <- '\x00'
+				var tree GitTree
+				tree, err = parseGitTree(data_in.ch)
+
+				if err != nil {
+					err = fmt.Errorf("Error parsing git tree: %w", err)
+					return
+				}
+
+				for _, te := range tree.items {
+					if te.isTree() {
+						directoryList[p+te.name] = te.hash
+					}
+				}
+			}
+		}
+	}()
+
+	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
+	cmd.Env = []string{
+		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
+		"GIT_LFS_SKIP_SMUDGE=1",
+		"GIT_CONFIG_GLOBAL=/dev/null",
+	}
+	cmd.Dir = filepath.Join(e.GitPath, gitPath)
+	cmd.Stdout = &data_in
+	cmd.Stdin = &data_out
+	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
+		LogError(string(data))
+		return len(data), nil
+	})
+	LogDebug("command run:", cmd.Args)
+	if e := cmd.Run(); e != nil {
+		LogError(e)
+		close(data_in.ch)
+		close(data_out.ch)
+		return directoryList, e
+	}
+
+	done.Lock()
+	return directoryList, err
+}
+
 // return (filename) -> (hash) map for all submodules
 func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleList map[string]string, err error) {
 	var done sync.Mutex