colin
/
tarballer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add verbose mode flag for detailed output during extraction and creation

This commit is contained in:
Leopere 2025-03-20 22:30:55 -04:00
parent f1a25f959e
commit b546e84afd
4 changed files with 227 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
bin/tarballer-darwin
View File

Binary file not shown.

BIN
bin/tarballer-freebsd
View File

Binary file not shown.

BIN
bin/tarballer-linux
View File

Binary file not shown.

305
main.go
View File

@ -12,6 +12,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
)
const manifestFilename = ".md5-manifest.txt"
@ -24,6 +25,7 @@ func main() {
extractMode := flag.Bool("extract", false, "Extract mode (instead of create)")
extractDir := flag.String("extractdir", "", "Directory to extract to (default: current directory)")
verifyOnly := flag.Bool("verify", false, "Only verify hash integrity without extraction")
verboseMode := flag.Bool("verbose", false, "Enable verbose output")
flag.Parse()
if *extractMode {
@ -43,7 +45,7 @@ func main() {
extractTo = "."
}
err := extractTarball(*outputFile, extractTo, *verifyOnly)
err := extractTarball(*outputFile, extractTo, *verifyOnly, *verboseMode)
if err != nil {
fmt.Printf("Error extracting tarball: %v\n", err)
os.Exit(1)
@ -56,7 +58,7 @@ func main() {
}
err := createTarball(*sourceDir, *outputFile, *prefixDir)
err := createTarball(*sourceDir, *outputFile, *prefixDir, *verboseMode)
if err != nil {
fmt.Printf("Error creating tarball: %v\n", err)
os.Exit(1)
@ -82,8 +84,22 @@ func calcFileMD5(filePath string) (string, error) {
return hex.EncodeToString(hash.Sum(nil)), nil
}
func createTarball(sourceDir, outputFile, prefix string) error {
// Create output file
func createTarball(sourceDir, outputFile, prefix string, verboseMode bool) error {
// Resolve absolute path of source directory
absSourceDir, err := filepath.Abs(sourceDir)
if err != nil {
return err
}
if verboseMode {
fmt.Printf("Creating tarball from directory: %s\n", absSourceDir)
fmt.Printf("Output file: %s\n", outputFile)
if prefix != "" {
fmt.Printf("Using prefix: %s\n", prefix)
}
}
// Create the output file
out, err := os.Create(outputFile)
if err != nil {
return err
@ -91,6 +107,9 @@ func createTarball(sourceDir, outputFile, prefix string) error {
defer out.Close()
// Create gzip writer
if verboseMode {
fmt.Println("Creating compressed archive...")
}
gw := gzip.NewWriter(out)
defer gw.Close()
@ -98,93 +117,114 @@ func createTarball(sourceDir, outputFile, prefix string) error {
tw := tar.NewWriter(gw)
defer tw.Close()
// Create a map to store MD5 hashes
fileHashes := make(map[string]string)
// Create a map to store file hashes
hashes := make(map[string]string)
fileCount := 0
// Resolve absolute source path to handle relative symlinks correctly
sourceDir, err = filepath.Abs(sourceDir)
if err != nil {
return err
}
// Walk through source directory
err = filepath.Walk(sourceDir, func(filePath string, info os.FileInfo, err error) error {
// Walk through the source directory
err = filepath.Walk(absSourceDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Get relative path to use in the tarball
relPath, err := filepath.Rel(sourceDir, filePath)
fileCount++
if verboseMode && fileCount%100 == 0 {
fmt.Printf("Processed %d files...\n", fileCount)
}
// Get the relative path
relPath, err := filepath.Rel(absSourceDir, path)
if err != nil {
return err
}
// Skip the manifest file if it exists (from a previous run)
if relPath == manifestFilename {
// Skip the root directory
if relPath == "." {
return nil
}
// Create tar header using original file info
// Add prefix if specified
if prefix != "" {
relPath = filepath.Join(prefix, relPath)
}
// Create header
header, err := tar.FileInfoHeader(info, "")
if err != nil {
return err
}
// Update the name with the prefix and relative path
header.Name = filepath.Join(prefix, relPath)
// Set the name to be the relative path (with prefix if specified)
header.Name = filepath.ToSlash(relPath)
// Special handling for symbolic links
// Handle symlinks
if info.Mode()&os.ModeSymlink != 0 {
// Read link target
linkTarget, err := os.Readlink(filePath)
linkTarget, err := os.Readlink(path)
if err != nil {
return err
}
// Store the link target in the header
header.Linkname = linkTarget
// Make sure the link type is set correctly
header.Typeflag = tar.TypeSymlink
// Write header
if err := tw.WriteHeader(header); err != nil {
return err
}
// No content to write for symlinks
} else if !info.IsDir() {
// Regular file - write header first
if err := tw.WriteHeader(header); err != nil {
return err
}
// Open the file for reading
file, err := os.Open(filePath)
if err != nil {
return err
}
// Create a multiwriter to write to both the tar archive and MD5 hash
hashWriter := md5.New()
multiWriter := io.MultiWriter(tw, hashWriter)
// Copy file contents to both the tar archive and hash calculator
_, err = io.Copy(multiWriter, file)
file.Close() // Close file after reading
if err != nil {
return err
}
// Store the calculated hash in our map
hashString := hex.EncodeToString(hashWriter.Sum(nil))
fileHashes[header.Name] = hashString
} else {
// For directories, just write the header
if verboseMode {
fmt.Printf("Adding symlink: %s -> %s\n", header.Name, linkTarget)
}
// Write the header to the tarball
if err := tw.WriteHeader(header); err != nil {
return err
}
return nil
}
// Skip directories in hash calculation, but include in tarball
if info.IsDir() {
if verboseMode {
fmt.Printf("Adding directory: %s\n", header.Name)
}
// Write the header to the tarball
if err := tw.WriteHeader(header); err != nil {
return err
}
return nil
}
// Skip the manifest file if it already exists in the source dir
if filepath.Base(path) == manifestFilename && filepath.Dir(path) == absSourceDir {
if verboseMode {
fmt.Printf("Skipping existing manifest file: %s\n", path)
}
return nil
}
if verboseMode {
fmt.Printf("Adding file: %s\n", header.Name)
}
// Write the header to the tarball
if err := tw.WriteHeader(header); err != nil {
return err
}
// Open the file for reading
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// Create a hash writer to calculate MD5 while copying
h := md5.New()
multiWriter := io.MultiWriter(tw, h)
// Copy file content to both the tarball and hash function
if _, err := io.Copy(multiWriter, file); err != nil {
return err
}
// Store the hash
hashes[header.Name] = hex.EncodeToString(h.Sum(nil))
return nil
})
@ -192,35 +232,62 @@ func createTarball(sourceDir, outputFile, prefix string) error {
return err
}
// Create and add the manifest file
var manifestContent strings.Builder
for path, hash := range fileHashes {
manifestContent.WriteString(fmt.Sprintf("%s %s\n", hash, path))
if verboseMode {
fmt.Printf("Added %d files to tarball\n", fileCount)
fmt.Println("Creating MD5 manifest...")
}
// Create a tar header for the manifest
manifestHeader := &tar.Header{
Name: manifestFilename,
Mode: 0644,
Size: int64(manifestContent.Len()),
Typeflag: tar.TypeReg,
// Create MD5 manifest
var manifest strings.Builder
for path, hash := range hashes {
fmt.Fprintf(&manifest, "%s %s\n", hash, path)
}
// Write the manifest header
if err := tw.WriteHeader(manifestHeader); err != nil {
// Create header for the manifest
header := &tar.Header{
Name: manifestFilename,
Mode: 0644,
Size: int64(manifest.Len()),
ModTime: time.Now(),
}
if verboseMode {
fmt.Printf("Adding manifest with %d entries\n", len(hashes))
}
// Write the manifest header to the tarball
if err := tw.WriteHeader(header); err != nil {
return err
}
// Write the manifest content
if _, err := tw.Write([]byte(manifestContent.String())); err != nil {
// Write the manifest content to the tarball
if _, err := io.WriteString(tw, manifest.String()); err != nil {
return err
}
// Close the writers
if err := tw.Close(); err != nil {
return err
}
if err := gw.Close(); err != nil {
return err
}
if verboseMode {
fmt.Printf("Tarball created successfully: %s\n", outputFile)
} else {
fmt.Println("Tarball created successfully!")
}
return nil
}
func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
func extractTarball(tarballPath, extractDir string, verifyOnly, verboseMode bool) error {
// Open the tarball
if verboseMode {
fmt.Printf("Opening tarball: %s\n", tarballPath)
}
file, err := os.Open(tarballPath)
if err != nil {
return err
@ -243,6 +310,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
tempDir := ""
if !verifyOnly {
// Create a temporary directory for extraction
if verboseMode {
fmt.Printf("Creating temporary extraction directory in: %s\n", extractDir)
}
tempDir, err = os.MkdirTemp(extractDir, "tarballer-extract-")
if err != nil {
return err
@ -250,6 +320,12 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
}
// Extract files to get the manifest
fileCount := 0
if verboseMode {
fmt.Println("First pass: Looking for manifest file...")
}
for {
header, err := tr.Next()
if err == io.EOF {
@ -267,8 +343,17 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
continue
}
fileCount++
if verboseMode && fileCount%100 == 0 {
fmt.Printf("Processed %d files while searching for manifest...\n", fileCount)
}
// Check if this is the manifest file
if filepath.Base(header.Name) == manifestFilename && filepath.Dir(header.Name) == "." {
if verboseMode {
fmt.Println("Found manifest file, parsing hashes...")
}
// Read the manifest content
var content strings.Builder
if _, err := io.Copy(&content, tr); err != nil {
@ -279,6 +364,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
}
// Parse the manifest to get expected hashes
hashCount := 0
scanner := bufio.NewScanner(strings.NewReader(content.String()))
for scanner.Scan() {
line := scanner.Text()
@ -287,9 +373,14 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
hash := parts[0]
path := parts[1]
expectedHashes[path] = hash
hashCount++
}
}
if verboseMode {
fmt.Printf("Parsed %d hash entries from manifest\n", hashCount)
}
if err := scanner.Err(); err != nil {
if tempDir != "" {
os.RemoveAll(tempDir)
@ -308,6 +399,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Extract to temp dir to verify hashes
target := filepath.Join(tempDir, header.Name)
if verboseMode {
fmt.Printf("Extracting (first pass): %s\n", header.Name)
}
// Create directory if needed
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(target, 0755); err != nil {
@ -325,6 +420,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Handle symlinks
if header.Typeflag == tar.TypeSymlink {
if verboseMode {
fmt.Printf("Creating symlink: %s -> %s\n", target, header.Linkname)
}
if err := os.Symlink(header.Linkname, target); err != nil {
os.RemoveAll(tempDir)
return err
@ -358,6 +456,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// If we're only verifying, we need to reopen the tarball
if verifyOnly {
if verboseMode {
fmt.Println("Reopening tarball for verification...")
}
file.Seek(0, 0)
gr, err = gzip.NewReader(file)
if err != nil {
@ -372,8 +473,15 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
fileVerified := make(map[string]bool)
missingFiles := []string{}
if verboseMode {
fmt.Println("Second pass: Verifying file integrity...")
}
if verifyOnly {
// Extract to temp dir for verification
if verboseMode {
fmt.Println("Creating temporary directory for verification only...")
}
tempDir, err = os.MkdirTemp(extractDir, "tarballer-verify-")
if err != nil {
return err
@ -381,6 +489,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
defer os.RemoveAll(tempDir)
}
fileCount = 0
for {
header, err := tr.Next()
if err == io.EOF {
@ -393,6 +502,11 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
return err
}
fileCount++
if verboseMode && fileCount%100 == 0 {
fmt.Printf("Verified %d files...\n", fileCount)
}
// Skip directories and the manifest file for verification
if header.Typeflag == tar.TypeDir || (filepath.Base(header.Name) == manifestFilename && filepath.Dir(header.Name) == ".") {
continue
@ -406,7 +520,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Check if this file has an expected hash
expectedHash, exists := expectedHashes[header.Name]
if !exists {
fmt.Printf("Warning: File %s not found in manifest\n", header.Name)
if verboseMode {
fmt.Printf("Warning: File %s not found in manifest\n", header.Name)
}
continue
}
@ -414,6 +530,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
if verifyOnly {
target := filepath.Join(tempDir, header.Name)
if verboseMode {
fmt.Printf("Extracting for verification: %s\n", header.Name)
}
// Create parent directory if it doesn't exist
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
@ -443,6 +563,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
fmt.Printf("Hash mismatch for %s: expected %s, got %s\n", header.Name, expectedHash, actualHash)
verificationFailed = true
} else {
if verboseMode {
fmt.Printf("Hash verified: %s\n", header.Name)
}
fileVerified[header.Name] = true
}
} else {
@ -459,6 +582,9 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
fmt.Printf("Hash mismatch for %s: expected %s, got %s\n", header.Name, expectedHash, actualHash)
verificationFailed = true
} else {
if verboseMode {
fmt.Printf("Hash verified: %s\n", header.Name)
}
fileVerified[header.Name] = true
}
}
@ -504,6 +630,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
}
// Move the extracted files to the final destination (excluding manifest if needed)
if verboseMode {
fmt.Println("Moving verified files to final destination...")
}
files, err := os.ReadDir(tempDir)
if err != nil {
os.RemoveAll(tempDir)
@ -516,6 +646,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
return err
}
movedCount := 0
// Move each top-level extracted item
for _, f := range files {
source := filepath.Join(tempDir, f.Name())
@ -523,9 +654,16 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Skip the manifest file if needed
if f.Name() == manifestFilename {
if verboseMode {
fmt.Println("Skipping manifest file in final destination")
}
continue
}
if verboseMode {
fmt.Printf("Moving: %s -> %s\n", source, dest)
}
// If destination already exists, remove it
if _, err := os.Stat(dest); err == nil {
if err := os.RemoveAll(dest); err != nil {
@ -537,6 +675,10 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Move the file
if err := os.Rename(source, dest); err != nil {
// If rename fails (e.g., across devices), try copying
if verboseMode {
fmt.Printf("Direct move failed, using recursive copy for: %s\n", f.Name())
}
err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
@ -588,11 +730,18 @@ func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
return err
}
}
movedCount++
}
// Clean up temp directory
os.RemoveAll(tempDir)
fmt.Println("Extraction completed and verified successfully!")
if verboseMode {
fmt.Printf("Extraction complete: %d files extracted and verified, %d files moved to final destination\n", fileCount, movedCount)
} else {
fmt.Println("Extraction completed and verified successfully!")
}
return nil
}