diff --git a/Makefile b/Makefile index efd46a2..5951f14 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-staging-bot obs-status-service workflow-direct workflow-pr +MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-groups-bot obs-staging-bot obs-status-service workflow-direct workflow-pr build: for m in $(MODULES); do go build -C $$m -buildmode=pie || exit 1 ; done diff --git a/obs-groups-bot/.gitignore b/obs-groups-bot/.gitignore new file mode 100644 index 0000000..f45db06 --- /dev/null +++ b/obs-groups-bot/.gitignore @@ -0,0 +1 @@ +obs-groups-bot diff --git a/obs-groups-bot/main.go b/obs-groups-bot/main.go new file mode 100644 index 0000000..bbe1dba --- /dev/null +++ b/obs-groups-bot/main.go @@ -0,0 +1,242 @@ +// Connect to the Open Build Service (OBS) API, retrieves a list of all groups, +// and exports their metadata (specifically member lists) into individual JSON files. +// +// The tool supports both command-line flags and environment variables for configuration +// (not for authentication, which is only via env vars), and includes a debug mode for verbose output. +// It handles different XML response formats from the OBS API and ensures that +// the output JSON files are properly sanitized and formatted. +// +// The accepted command-line flags are: +// +// -debug: Enable debug output showing API URLs and responses. +// -instance: Name of the OBS instance (used in metadata, default "openSUSE"). +// -host: Base URL of the OBS API (default "http://localhost:3000"). +// -output: Directory to save the JSON files (default "groups"). +// +// Usage: +// +// # Using environment variables (OBS_USER, OBS_PASSWORD) +// go run main.go +// +// # Targeting a specific OBS instance and output directory +// go run main.go -host "https://api.opensuse.org" -output "./obs_groups" +// +// # Full command with debug mode +// go run main.go -host http://localhost:8000 -output "./obs_groups" -instance "OBS" -debug + +package main + +import ( + "encoding/json" + "encoding/xml" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "time" + + "src.opensuse.org/autogits/common" +) + +type groupsList struct { + XMLName xml.Name `xml:"groups"` + Groups []groupItem `xml:"group"` +} + +type groupsListAlt struct { + XMLName xml.Name `xml:"directory"` + Entries []groupEntry `xml:"entry"` +} + +type groupEntry struct { + Name string `xml:"name,attr,omitempty"` + Inner string `xml:",innerxml"` +} + +func (e *groupEntry) getName() string { + if e.Name != "" { + return e.Name + } + return e.Inner +} + +type groupItem struct { + GroupID string `xml:"groupid,attr"` +} + +func getAllGroups(client *common.ObsClient) ([]string, error) { + res, err := client.ObsRequest("GET", []string{"group"}, nil) + if err != nil { + return nil, err + } + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + + log.Printf("Response status: %d, body length: %d", res.StatusCode, len(data)) + if res.StatusCode != 200 { + bodyStr := string(data) + if len(bodyStr) > 500 { + bodyStr = bodyStr[:500] + } + return nil, fmt.Errorf("Unexpected return code: %d, body: %s", res.StatusCode, bodyStr) + } + + // Try parsing as format + var groupsList groupsList + err = xml.Unmarshal(data, &groupsList) + if err == nil && len(groupsList.Groups) > 0 { + groupIDs := make([]string, len(groupsList.Groups)) + for i, g := range groupsList.Groups { + groupIDs[i] = g.GroupID + } + return groupIDs, nil + } + + // Try parsing as format + var groupsAlt groupsListAlt + err = xml.Unmarshal(data, &groupsAlt) + if err == nil && len(groupsAlt.Entries) > 0 { + groupIDs := make([]string, len(groupsAlt.Entries)) + for i, e := range groupsAlt.Entries { + groupIDs[i] = e.getName() + } + return groupIDs, nil + } + + // Log what we got + bodyStr := string(data) + if len(bodyStr) > 1000 { + bodyStr = bodyStr[:1000] + } + log.Printf("Failed to parse XML, got: %s", bodyStr) + return nil, fmt.Errorf("Could not parse groups response") +} + +type GroupOutput struct { + Meta ImportMeta `json:"_meta,omitempty"` + Name string `json:"Name"` + Reviewers []string `json:"Reviewers"` + Silent bool `json:"Silent,omitempty"` +} + +type ImportMeta struct { + ImportedFrom string `json:"imported_from"` + ReadOnly bool `json:"read_only"` + ImportTime time.Time `json:"import_time"` +} + +func sanitizeFilename(name string) string { + name = strings.ReplaceAll(name, "/", "_") + name = strings.ReplaceAll(name, ":", "_") + name = strings.ReplaceAll(name, " ", "_") + return name +} + +func processGroup(client *common.ObsClient, groupID, outputDir, instanceName string, importTime time.Time) error { + meta, err := client.GetGroupMeta(groupID) + if err != nil { + return fmt.Errorf("fetching group meta: %w", err) + } + + if meta == nil { + return fmt.Errorf("group not found") + } + + common.LogDebug(fmt.Sprintf("Group meta for %s: Title: %s, Persons: %d", groupID, meta.Title, len(meta.Persons.Persons))) + + reviewers := make([]string, 0, len(meta.Persons.Persons)) + for _, p := range meta.Persons.Persons { + reviewers = append(reviewers, p.UserID) + } + + output := GroupOutput{ + Meta: ImportMeta{ + ImportedFrom: instanceName, + ReadOnly: true, + ImportTime: importTime, + }, + Name: groupID, + Reviewers: reviewers, + } + + filename := sanitizeFilename(groupID) + ".json" + filePath := filepath.Join(outputDir, filename) + + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("marshaling json: %w", err) + } + + if err := os.WriteFile(filePath, data, 0644); err != nil { + return fmt.Errorf("writing file: %w", err) + } + + common.LogDebug(fmt.Sprintf("Saved group %s to %s", groupID, filePath)) + + return nil +} + +func main() { + debugModePtr := flag.Bool("debug", false, "Enable debug output showing API URLs") + obsInstance := flag.String("instance", "openSUSE", "OBS instance name (used in metadata)") + obsHost := flag.String("host", "http://localhost:3000", "OBS API host URL") + outputDir := flag.String("output", "groups", "Output directory for JSON files") + flag.Parse() + + if *debugModePtr { + common.SetLoggingLevel(common.LogLevelDebug) + } + + if err := common.RequireObsSecretToken(); err != nil { + log.Fatal(err) + } + + log.Printf("Connecting to OBS at %s (instance: %s)", *obsHost, *obsInstance) + + client, err := common.NewObsClient(*obsHost) + if err != nil { + log.Fatalf("Failed to create OBS client: %v", err) + } + + log.Println("Fetching list of all groups...") + groupIDs, err := getAllGroups(client) + if err != nil { + log.Fatalf("Failed to get groups list: %v", err) + } + + log.Printf("Found %d groups: %v", len(groupIDs), groupIDs) + log.Printf("Found %s ", groupIDs) + + err = os.MkdirAll(*outputDir, 0755) + if err != nil { + log.Fatalf("Failed to create output directory: %v", err) + } + + importTime := time.Now() + + successCount := 0 + errorCount := 0 + + for i, groupID := range groupIDs { + log.Printf("[%d/%d] Fetching group: %s", i+1, len(groupIDs), groupID) + + if err := processGroup(client, groupID, *outputDir, *obsInstance, importTime); err != nil { + log.Printf("Error processing group %s: %v", groupID, err) + errorCount++ + continue + } + + successCount++ + time.Sleep(100 * time.Millisecond) + } + + log.Printf("Done! Success: %d, Errors: %d", successCount, errorCount) + log.Printf("JSON files saved to: %s", *outputDir) +} diff --git a/obs-groups-bot/main_test.go b/obs-groups-bot/main_test.go new file mode 100644 index 0000000..470232d --- /dev/null +++ b/obs-groups-bot/main_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "encoding/json" + "encoding/xml" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "src.opensuse.org/autogits/common" +) + +func TestGroupsListParsing(t *testing.T) { + // Test format + groupsXML := ` + + + + +` + + var groupsList groupsList + err := xml.Unmarshal([]byte(groupsXML), &groupsList) + if err != nil { + t.Fatalf("Failed to unmarshal groups XML: %v", err) + } + + if len(groupsList.Groups) != 3 { + t.Errorf("Expected 3 groups, got %d", len(groupsList.Groups)) + } + + expected := []string{"group1", "group2", "group3"} + for i, g := range groupsList.Groups { + if g.GroupID != expected[i] { + t.Errorf("Expected group %s, got %s", expected[i], g.GroupID) + } + } +} + +func TestProcessGroup(t *testing.T) { + // 1. Mock the OBS API server for GetGroupMeta + groupID := "test:group" + mockGroupMetaResponse := ` + + Test Group Title + + + + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedPath := "/group/" + groupID + if r.URL.Path != expectedPath { + t.Errorf("Expected path %s, got %s", expectedPath, r.URL.Path) + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockGroupMetaResponse)) + })) + defer server.Close() + + // 2. Create a temporary directory for output + outputDir := t.TempDir() + + // 3. Initialize client pointing to mock server + client, err := common.NewObsClient(server.URL) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + // 4. Call processGroup + instanceName := "test-instance" + importTime := time.Now().UTC().Truncate(time.Second) // Truncate for stable comparison + err = processGroup(client, groupID, outputDir, instanceName, importTime) + if err != nil { + t.Fatalf("processGroup failed: %v", err) + } + + // 5. Verify the output file + expectedFilename := sanitizeFilename(groupID) + ".json" + filePath := filepath.Join(outputDir, expectedFilename) + + // Check if file exists + if _, err := os.Stat(filePath); os.IsNotExist(err) { + t.Fatalf("Expected output file was not created: %s", filePath) + } + + // Read and verify file content + data, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read output file: %v", err) + } + + var result GroupOutput + if err := json.Unmarshal(data, &result); err != nil { + t.Fatalf("Failed to unmarshal output JSON: %v", err) + } + + // Assertions + expectedReviewers := []string{"user1", "user2"} + expectedOutput := GroupOutput{ + Meta: ImportMeta{ + ImportedFrom: instanceName, + ReadOnly: true, + ImportTime: importTime, + }, + Name: groupID, + Reviewers: expectedReviewers, + } + + // Use reflect.DeepEqual for a robust comparison of the structs + if !reflect.DeepEqual(result, expectedOutput) { + t.Errorf("Output JSON does not match expected.\nGot: %+v\nWant: %+v", result, expectedOutput) + } +} + +func TestGetAllGroups(t *testing.T) { + // Mock the OBS API server + mockResponse := ` + + + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify the request path + if r.URL.Path != "/group" { + t.Errorf("Expected path /group, got %s", r.URL.Path) + } + // Verify method + if r.Method != "GET" { + t.Errorf("Expected method GET, got %s", r.Method) + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockResponse)) + })) + defer server.Close() + + // Initialize client pointing to mock server + client, err := common.NewObsClient(server.URL) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + groups, err := getAllGroups(client) + if err != nil { + t.Fatalf("GetAllGroups failed: %v", err) + } + + if len(groups) != 2 { + t.Errorf("Expected 2 groups, got %d", len(groups)) + } + if groups[0] != "mock-group-1" { + t.Errorf("Expected first group to be mock-group-1, got %s", groups[0]) + } +} + +func TestGroupsListDirectoryFormat(t *testing.T) { + // Test format with name attribute + dirXML := ` + + + + +` + + var groupsAlt groupsListAlt + err := xml.Unmarshal([]byte(dirXML), &groupsAlt) + if err != nil { + t.Fatalf("Failed to unmarshal directory XML: %v", err) + } + + if len(groupsAlt.Entries) != 3 { + t.Errorf("Expected 3 entries, got %d", len(groupsAlt.Entries)) + } + + expected := []string{"group-a", "group-b", "group-c"} + for i, e := range groupsAlt.Entries { + if e.getName() != expected[i] { + t.Errorf("Expected entry %s, got %s", expected[i], e.getName()) + } + } +} + +func TestSanitizeFilename(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"simple", "simple"}, + {"group/name", "group_name"}, + {"project:group", "project_group"}, + {"group with spaces", "group_with_spaces"}, + {"group/name:space", "group_name_space"}, + {"", ""}, + {"multiple///slashes", "multiple___slashes"}, + } + + for _, tc := range tests { + result := sanitizeFilename(tc.input) + if result != tc.expected { + t.Errorf("sanitizeFilename(%q) = %q, expected %q", tc.input, result, tc.expected) + } + } +}