diff --git a/bin/tarballer-darwin b/bin/tarballer-darwin index 34375a3..d962ec6 100755 Binary files a/bin/tarballer-darwin and b/bin/tarballer-darwin differ diff --git a/bin/tarballer-freebsd b/bin/tarballer-freebsd index f363797..8654014 100755 Binary files a/bin/tarballer-freebsd and b/bin/tarballer-freebsd differ diff --git a/bin/tarballer-linux b/bin/tarballer-linux index dbe5733..fb85b47 100755 Binary files a/bin/tarballer-linux and b/bin/tarballer-linux differ diff --git a/main.go b/main.go index fd82926..e0c4de4 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "strings" + "time" ) const manifestFilename = ".md5-manifest.txt" @@ -24,6 +25,7 @@ func main() { extractMode := flag.Bool("extract", false, "Extract mode (instead of create)") extractDir := flag.String("extractdir", "", "Directory to extract to (default: current directory)") verifyOnly := flag.Bool("verify", false, "Only verify hash integrity without extraction") + verboseMode := flag.Bool("verbose", false, "Enable verbose output") flag.Parse() if *extractMode { @@ -43,7 +45,7 @@ func main() { extractTo = "." } - err := extractTarball(*outputFile, extractTo, *verifyOnly) + err := extractTarball(*outputFile, extractTo, *verifyOnly, *verboseMode) if err != nil { fmt.Printf("Error extracting tarball: %v\n", err) os.Exit(1) @@ -56,7 +58,7 @@ func main() { } - err := createTarball(*sourceDir, *outputFile, *prefixDir) + err := createTarball(*sourceDir, *outputFile, *prefixDir, *verboseMode) if err != nil { fmt.Printf("Error creating tarball: %v\n", err) os.Exit(1) @@ -82,8 +84,22 @@ func calcFileMD5(filePath string) (string, error) { return hex.EncodeToString(hash.Sum(nil)), nil } -func createTarball(sourceDir, outputFile, prefix string) error { - // Create output file +func createTarball(sourceDir, outputFile, prefix string, verboseMode bool) error { + // Resolve absolute path of source directory + absSourceDir, err := filepath.Abs(sourceDir) + if err != nil { + return err + } + + if verboseMode { + fmt.Printf("Creating tarball from directory: %s\n", absSourceDir) + fmt.Printf("Output file: %s\n", outputFile) + if prefix != "" { + fmt.Printf("Using prefix: %s\n", prefix) + } + } + + // Create the output file out, err := os.Create(outputFile) if err != nil { return err @@ -91,6 +107,9 @@ func createTarball(sourceDir, outputFile, prefix string) error { defer out.Close() // Create gzip writer + if verboseMode { + fmt.Println("Creating compressed archive...") + } gw := gzip.NewWriter(out) defer gw.Close() @@ -98,93 +117,114 @@ func createTarball(sourceDir, outputFile, prefix string) error { tw := tar.NewWriter(gw) defer tw.Close() - // Create a map to store MD5 hashes - fileHashes := make(map[string]string) + // Create a map to store file hashes + hashes := make(map[string]string) + fileCount := 0 - // Resolve absolute source path to handle relative symlinks correctly - sourceDir, err = filepath.Abs(sourceDir) - if err != nil { - return err - } - - // Walk through source directory - err = filepath.Walk(sourceDir, func(filePath string, info os.FileInfo, err error) error { + // Walk through the source directory + err = filepath.Walk(absSourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - // Get relative path to use in the tarball - relPath, err := filepath.Rel(sourceDir, filePath) + fileCount++ + if verboseMode && fileCount%100 == 0 { + fmt.Printf("Processed %d files...\n", fileCount) + } + + // Get the relative path + relPath, err := filepath.Rel(absSourceDir, path) if err != nil { return err } - // Skip the manifest file if it exists (from a previous run) - if relPath == manifestFilename { + // Skip the root directory + if relPath == "." { return nil } - // Create tar header using original file info + // Add prefix if specified + if prefix != "" { + relPath = filepath.Join(prefix, relPath) + } + + // Create header header, err := tar.FileInfoHeader(info, "") if err != nil { return err } - // Update the name with the prefix and relative path - header.Name = filepath.Join(prefix, relPath) + // Set the name to be the relative path (with prefix if specified) + header.Name = filepath.ToSlash(relPath) - // Special handling for symbolic links + // Handle symlinks if info.Mode()&os.ModeSymlink != 0 { - // Read link target - linkTarget, err := os.Readlink(filePath) + linkTarget, err := os.Readlink(path) if err != nil { return err } - - // Store the link target in the header header.Linkname = linkTarget - - // Make sure the link type is set correctly header.Typeflag = tar.TypeSymlink - // Write header - if err := tw.WriteHeader(header); err != nil { - return err - } - // No content to write for symlinks - } else if !info.IsDir() { - // Regular file - write header first - if err := tw.WriteHeader(header); err != nil { - return err - } - - // Open the file for reading - file, err := os.Open(filePath) - if err != nil { - return err - } - - // Create a multiwriter to write to both the tar archive and MD5 hash - hashWriter := md5.New() - multiWriter := io.MultiWriter(tw, hashWriter) - - // Copy file contents to both the tar archive and hash calculator - _, err = io.Copy(multiWriter, file) - file.Close() // Close file after reading - if err != nil { - return err - } - - // Store the calculated hash in our map - hashString := hex.EncodeToString(hashWriter.Sum(nil)) - fileHashes[header.Name] = hashString - } else { - // For directories, just write the header + if verboseMode { + fmt.Printf("Adding symlink: %s -> %s\n", header.Name, linkTarget) + } + + // Write the header to the tarball if err := tw.WriteHeader(header); err != nil { return err } + return nil } + // Skip directories in hash calculation, but include in tarball + if info.IsDir() { + if verboseMode { + fmt.Printf("Adding directory: %s\n", header.Name) + } + // Write the header to the tarball + if err := tw.WriteHeader(header); err != nil { + return err + } + return nil + } + + // Skip the manifest file if it already exists in the source dir + if filepath.Base(path) == manifestFilename && filepath.Dir(path) == absSourceDir { + if verboseMode { + fmt.Printf("Skipping existing manifest file: %s\n", path) + } + return nil + } + + if verboseMode { + fmt.Printf("Adding file: %s\n", header.Name) + } + + // Write the header to the tarball + if err := tw.WriteHeader(header); err != nil { + return err + } + + // Open the file for reading + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Create a hash writer to calculate MD5 while copying + h := md5.New() + multiWriter := io.MultiWriter(tw, h) + + // Copy file content to both the tarball and hash function + if _, err := io.Copy(multiWriter, file); err != nil { + return err + } + + // Store the hash + hashes[header.Name] = hex.EncodeToString(h.Sum(nil)) + return nil }) @@ -192,35 +232,62 @@ func createTarball(sourceDir, outputFile, prefix string) error { return err } - // Create and add the manifest file - var manifestContent strings.Builder - for path, hash := range fileHashes { - manifestContent.WriteString(fmt.Sprintf("%s %s\n", hash, path)) + if verboseMode { + fmt.Printf("Added %d files to tarball\n", fileCount) + fmt.Println("Creating MD5 manifest...") } - // Create a tar header for the manifest - manifestHeader := &tar.Header{ - Name: manifestFilename, - Mode: 0644, - Size: int64(manifestContent.Len()), - Typeflag: tar.TypeReg, + // Create MD5 manifest + var manifest strings.Builder + for path, hash := range hashes { + fmt.Fprintf(&manifest, "%s %s\n", hash, path) } - // Write the manifest header - if err := tw.WriteHeader(manifestHeader); err != nil { + // Create header for the manifest + header := &tar.Header{ + Name: manifestFilename, + Mode: 0644, + Size: int64(manifest.Len()), + ModTime: time.Now(), + } + + if verboseMode { + fmt.Printf("Adding manifest with %d entries\n", len(hashes)) + } + + // Write the manifest header to the tarball + if err := tw.WriteHeader(header); err != nil { return err } - // Write the manifest content - if _, err := tw.Write([]byte(manifestContent.String())); err != nil { + // Write the manifest content to the tarball + if _, err := io.WriteString(tw, manifest.String()); err != nil { return err } + // Close the writers + if err := tw.Close(); err != nil { + return err + } + if err := gw.Close(); err != nil { + return err + } + + if verboseMode { + fmt.Printf("Tarball created successfully: %s\n", outputFile) + } else { + fmt.Println("Tarball created successfully!") + } + return nil } -func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { +func extractTarball(tarballPath, extractDir string, verifyOnly, verboseMode bool) error { // Open the tarball + if verboseMode { + fmt.Printf("Opening tarball: %s\n", tarballPath) + } + file, err := os.Open(tarballPath) if err != nil { return err @@ -243,6 +310,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { tempDir := "" if !verifyOnly { // Create a temporary directory for extraction + if verboseMode { + fmt.Printf("Creating temporary extraction directory in: %s\n", extractDir) + } tempDir, err = os.MkdirTemp(extractDir, "tarballer-extract-") if err != nil { return err @@ -250,6 +320,12 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { } // Extract files to get the manifest + fileCount := 0 + + if verboseMode { + fmt.Println("First pass: Looking for manifest file...") + } + for { header, err := tr.Next() if err == io.EOF { @@ -267,8 +343,17 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { continue } + fileCount++ + if verboseMode && fileCount%100 == 0 { + fmt.Printf("Processed %d files while searching for manifest...\n", fileCount) + } + // Check if this is the manifest file if filepath.Base(header.Name) == manifestFilename && filepath.Dir(header.Name) == "." { + if verboseMode { + fmt.Println("Found manifest file, parsing hashes...") + } + // Read the manifest content var content strings.Builder if _, err := io.Copy(&content, tr); err != nil { @@ -279,6 +364,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { } // Parse the manifest to get expected hashes + hashCount := 0 scanner := bufio.NewScanner(strings.NewReader(content.String())) for scanner.Scan() { line := scanner.Text() @@ -287,9 +373,14 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { hash := parts[0] path := parts[1] expectedHashes[path] = hash + hashCount++ } } + if verboseMode { + fmt.Printf("Parsed %d hash entries from manifest\n", hashCount) + } + if err := scanner.Err(); err != nil { if tempDir != "" { os.RemoveAll(tempDir) @@ -308,6 +399,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // Extract to temp dir to verify hashes target := filepath.Join(tempDir, header.Name) + if verboseMode { + fmt.Printf("Extracting (first pass): %s\n", header.Name) + } + // Create directory if needed if header.Typeflag == tar.TypeDir { if err := os.MkdirAll(target, 0755); err != nil { @@ -325,6 +420,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // Handle symlinks if header.Typeflag == tar.TypeSymlink { + if verboseMode { + fmt.Printf("Creating symlink: %s -> %s\n", target, header.Linkname) + } if err := os.Symlink(header.Linkname, target); err != nil { os.RemoveAll(tempDir) return err @@ -358,6 +456,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // If we're only verifying, we need to reopen the tarball if verifyOnly { + if verboseMode { + fmt.Println("Reopening tarball for verification...") + } file.Seek(0, 0) gr, err = gzip.NewReader(file) if err != nil { @@ -372,8 +473,15 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { fileVerified := make(map[string]bool) missingFiles := []string{} + if verboseMode { + fmt.Println("Second pass: Verifying file integrity...") + } + if verifyOnly { // Extract to temp dir for verification + if verboseMode { + fmt.Println("Creating temporary directory for verification only...") + } tempDir, err = os.MkdirTemp(extractDir, "tarballer-verify-") if err != nil { return err @@ -381,6 +489,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { defer os.RemoveAll(tempDir) } + fileCount = 0 for { header, err := tr.Next() if err == io.EOF { @@ -393,6 +502,11 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { return err } + fileCount++ + if verboseMode && fileCount%100 == 0 { + fmt.Printf("Verified %d files...\n", fileCount) + } + // Skip directories and the manifest file for verification if header.Typeflag == tar.TypeDir || (filepath.Base(header.Name) == manifestFilename && filepath.Dir(header.Name) == ".") { continue @@ -406,7 +520,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // Check if this file has an expected hash expectedHash, exists := expectedHashes[header.Name] if !exists { - fmt.Printf("Warning: File %s not found in manifest\n", header.Name) + if verboseMode { + fmt.Printf("Warning: File %s not found in manifest\n", header.Name) + } continue } @@ -414,6 +530,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { if verifyOnly { target := filepath.Join(tempDir, header.Name) + if verboseMode { + fmt.Printf("Extracting for verification: %s\n", header.Name) + } + // Create parent directory if it doesn't exist if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err @@ -443,6 +563,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { fmt.Printf("Hash mismatch for %s: expected %s, got %s\n", header.Name, expectedHash, actualHash) verificationFailed = true } else { + if verboseMode { + fmt.Printf("Hash verified: %s\n", header.Name) + } fileVerified[header.Name] = true } } else { @@ -459,6 +582,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { fmt.Printf("Hash mismatch for %s: expected %s, got %s\n", header.Name, expectedHash, actualHash) verificationFailed = true } else { + if verboseMode { + fmt.Printf("Hash verified: %s\n", header.Name) + } fileVerified[header.Name] = true } } @@ -504,6 +630,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { } // Move the extracted files to the final destination (excluding manifest if needed) + if verboseMode { + fmt.Println("Moving verified files to final destination...") + } + files, err := os.ReadDir(tempDir) if err != nil { os.RemoveAll(tempDir) @@ -516,6 +646,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { return err } + movedCount := 0 // Move each top-level extracted item for _, f := range files { source := filepath.Join(tempDir, f.Name()) @@ -523,9 +654,16 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // Skip the manifest file if needed if f.Name() == manifestFilename { + if verboseMode { + fmt.Println("Skipping manifest file in final destination") + } continue } + if verboseMode { + fmt.Printf("Moving: %s -> %s\n", source, dest) + } + // If destination already exists, remove it if _, err := os.Stat(dest); err == nil { if err := os.RemoveAll(dest); err != nil { @@ -537,6 +675,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { // Move the file if err := os.Rename(source, dest); err != nil { // If rename fails (e.g., across devices), try copying + if verboseMode { + fmt.Printf("Direct move failed, using recursive copy for: %s\n", f.Name()) + } + err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -588,11 +730,18 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error { return err } } + + movedCount++ } // Clean up temp directory os.RemoveAll(tempDir) - fmt.Println("Extraction completed and verified successfully!") + if verboseMode { + fmt.Printf("Extraction complete: %d files extracted and verified, %d files moved to final destination\n", fileCount, movedCount) + } else { + fmt.Println("Extraction completed and verified successfully!") + } + return nil }