Move test files to test-tmp directory which is git-ignored

This commit is contained in:
Leopere 2025-03-20 18:50:48 -04:00
parent fa02d33791
commit cff36fb6e6
10 changed files with 85 additions and 83 deletions

3
.gitignore vendored
View File

@ -8,4 +8,5 @@
/test/standard-test/
/test/standard-extracted/
/test/reference-extracted/
/test/*-checksums.txt
/test/*-checksums.txt
/test-tmp/

View File

@ -71,7 +71,6 @@ The usage is the same for all binaries:
- `-output`: The tarball to extract (required)
- `-extractdir`: Directory to extract to (defaults to current directory)
- `-verify`: Only verify hash integrity without extraction
- `-keepmanifest`: Keep the MD5 manifest file after extraction (defaults to removing it)
### Examples
@ -92,7 +91,7 @@ Tarballer includes built-in file integrity protection:
1. When creating a tarball, MD5 hashes are calculated for all files and stored in a hidden manifest file (`.md5-manifest.txt`) at the root of the extraction directory
2. During extraction, hashes are verified to ensure files haven't been corrupted or tampered with
3. The manifest file is automatically removed after extraction unless `-keepmanifest` is specified
3. The manifest file is automatically removed after extraction
4. If any file fails verification, the extraction is aborted with an error
This provides an extra layer of security and data integrity validation compared to standard tar tools.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -5,6 +5,7 @@ services:
volumes:
- ./bin:/output
- ./test:/test
- ./test-tmp:/test-tmp
- .:/workdir
working_dir: /workdir
command: sh -c "cp /app/tarballer-freebsd /output/ && cp /app/tarballer-darwin /output/ && cp /app/tarballer-linux /output/ && chmod +x /output/tarballer-* && /bin/test.sh all"
command: sh -c "cp /app/tarballer-freebsd /output/ && cp /app/tarballer-darwin /output/ && cp /app/tarballer-linux /output/ && chmod +x /output/tarballer-* && mkdir -p /test-tmp && /bin/test.sh all"

View File

@ -24,7 +24,6 @@ func main() {
extractMode := flag.Bool("extract", false, "Extract mode (instead of create)")
extractDir := flag.String("extractdir", "", "Directory to extract to (default: current directory)")
verifyOnly := flag.Bool("verify", false, "Only verify hash integrity without extraction")
keepManifest := flag.Bool("keepmanifest", false, "Keep the MD5 manifest file after extraction")
flag.Parse()
if *extractMode {
@ -44,7 +43,7 @@ func main() {
extractTo = "."
}
err := extractTarball(*outputFile, extractTo, *verifyOnly, *keepManifest)
err := extractTarball(*outputFile, extractTo, *verifyOnly)
if err != nil {
fmt.Printf("Error extracting tarball: %v\n", err)
os.Exit(1)
@ -210,7 +209,7 @@ func createTarball(sourceDir, outputFile, prefix string) error {
return nil
}
func extractTarball(tarballPath, extractDir string, verifyOnly, keepManifest bool) error {
func extractTarball(tarballPath, extractDir string, verifyOnly bool) error {
// Open the tarball
file, err := os.Open(tarballPath)
if err != nil {
@ -493,7 +492,7 @@ func extractTarball(tarballPath, extractDir string, verifyOnly, keepManifest boo
dest := filepath.Join(extractDir, f.Name())
// Skip the manifest file if needed
if !keepManifest && (f.Name() == manifestFilename) {
if f.Name() == manifestFilename {
continue
}

View File

@ -0,0 +1 @@
test file

View File

@ -0,0 +1 @@
test file

View File

@ -9,10 +9,10 @@ KEEP_TEMP_FILES=${2:-0}
cleanup_files() {
if [ "$KEEP_TEMP_FILES" -eq 0 ]; then
echo "=== CLEANING UP TEMPORARY FILES ==="
rm -rf /test/complex /test/complex-extracted
rm -f /test/complex-original-md5.txt /test/complex-extracted-md5.txt
rm -rf /test/standard-test /test/standard-extracted /test/reference-extracted
rm -f /test/original-checksums.txt /test/standard-checksums.txt /test/reference-checksums.txt
rm -rf /test-tmp/complex /test-tmp/complex-extracted
rm -f /test-tmp/complex-original-md5.txt /test-tmp/complex-extracted-md5.txt
rm -rf /test-tmp/standard-test /test-tmp/standard-extracted /test-tmp/reference-extracted
rm -f /test-tmp/original-checksums.txt /test-tmp/standard-checksums.txt /test-tmp/reference-checksums.txt
rm -f /workdir/complex.tar.gz /workdir/standard.tar.gz /workdir/reference.tar.gz
echo "Temporary files cleaned up"
else
@ -24,75 +24,75 @@ run_basic_test() {
echo "=== RUNNING BASIC TEST ==="
# Clean up existing test directories
rm -rf /test/complex /test/complex-extracted
rm -f /test/complex-original-md5.txt /test/complex-extracted-md5.txt
rm -rf /test-tmp/complex /test-tmp/complex-extracted
rm -f /test-tmp/complex-original-md5.txt /test-tmp/complex-extracted-md5.txt
# Create complex directory structure
mkdir -p /test/complex/dir1/subdir1/subsubdir1
mkdir -p /test/complex/dir1/subdir2
mkdir -p /test/complex/dir2/subdir1
mkdir -p /test-tmp/complex/dir1/subdir1/subsubdir1
mkdir -p /test-tmp/complex/dir1/subdir2
mkdir -p /test-tmp/complex/dir2/subdir1
# Create files at different levels
echo 'root level file' > /test/complex/rootfile.txt
echo 'level 1 file in dir1' > /test/complex/dir1/file1.txt
echo 'level 1 file in dir2' > /test/complex/dir2/file2.txt
echo 'level 2 file in subdir1' > /test/complex/dir1/subdir1/file3.txt
echo 'level 2 file in subdir2' > /test/complex/dir1/subdir2/file4.txt
echo 'level 3 file in subsubdir1' > /test/complex/dir1/subdir1/subsubdir1/file5.txt
echo 'root level file' > /test-tmp/complex/rootfile.txt
echo 'level 1 file in dir1' > /test-tmp/complex/dir1/file1.txt
echo 'level 1 file in dir2' > /test-tmp/complex/dir2/file2.txt
echo 'level 2 file in subdir1' > /test-tmp/complex/dir1/subdir1/file3.txt
echo 'level 2 file in subdir2' > /test-tmp/complex/dir1/subdir2/file4.txt
echo 'level 3 file in subsubdir1' > /test-tmp/complex/dir1/subdir1/subsubdir1/file5.txt
# Create a symbolic link with a relative path instead of absolute
cd /test/complex/dir2 && ln -s ../rootfile.txt symlink.txt && cd /workdir
cd /test-tmp/complex/dir2 && ln -s ../rootfile.txt symlink.txt && cd /workdir
# Calculate MD5 hashes of original files for verification
find /test/complex -type f | sort | xargs md5sum > /test/complex-original-md5.txt
find /test-tmp/complex -type f | sort | xargs md5sum > /test-tmp/complex-original-md5.txt
# Print the original structure for reference
echo '=== ORIGINAL DIRECTORY STRUCTURE ==='
find /test/complex -type f -o -type l | sort
find /test-tmp/complex -type f -o -type l | sort
# Create the tarball
/bin/tarballer -source /test/complex -output /workdir/complex.tar.gz -prefix complex-app
/bin/tarballer -source /test-tmp/complex -output /workdir/complex.tar.gz -prefix complex-app
# Extract the tarball
mkdir -p /test/complex-extracted
tar -xzf /workdir/complex.tar.gz -C /test/complex-extracted
mkdir -p /test-tmp/complex-extracted
tar -xzf /workdir/complex.tar.gz -C /test-tmp/complex-extracted
# Verify the extracted structure
echo '=== EXTRACTED DIRECTORY STRUCTURE ==='
find /test/complex-extracted -type f -o -type l | sort
find /test-tmp/complex-extracted -type f -o -type l | sort
# Calculate MD5 hashes of extracted files
find /test/complex-extracted -type f | sort | xargs md5sum > /test/complex-extracted-md5.txt
find /test-tmp/complex-extracted -type f | sort | xargs md5sum > /test-tmp/complex-extracted-md5.txt
# Compare file content
echo '=== VERIFYING FILE CONTENTS ==='
cat /test/complex/rootfile.txt
cat /test-tmp/complex/rootfile.txt
echo ' <-- Original: rootfile.txt'
cat /test/complex-extracted/complex-app/rootfile.txt
cat /test-tmp/complex-extracted/complex-app/rootfile.txt
echo ' <-- Extracted: rootfile.txt'
cat /test/complex/dir1/subdir1/subsubdir1/file5.txt
cat /test-tmp/complex/dir1/subdir1/subsubdir1/file5.txt
echo ' <-- Original: deep nested file5.txt'
cat /test/complex-extracted/complex-app/dir1/subdir1/subsubdir1/file5.txt
cat /test-tmp/complex-extracted/complex-app/dir1/subdir1/subsubdir1/file5.txt
echo ' <-- Extracted: deep nested file5.txt'
# Test symlink
echo '=== TESTING SYMLINK ==='
ls -la /test/complex/dir2/symlink.txt
ls -la /test/complex-extracted/complex-app/dir2/symlink.txt
ls -la /test-tmp/complex/dir2/symlink.txt
ls -la /test-tmp/complex-extracted/complex-app/dir2/symlink.txt
# Verify MD5 hashes
echo '=== MD5 HASH VERIFICATION ==='
echo 'Original file hashes:'
cat /test/complex-original-md5.txt
cat /test-tmp/complex-original-md5.txt
echo 'Extracted file hashes:'
cat /test/complex-extracted-md5.txt
cat /test-tmp/complex-extracted-md5.txt
# Verify hash comparison
echo '=== COMPARING FILE HASHES ==='
# Extract just file paths from original hashes
ORIG_FILES=$(cat /test/complex-original-md5.txt | awk '{print $2}' | sort)
ORIG_FILES=$(cat /test-tmp/complex-original-md5.txt | awk '{print $2}' | sort)
# For each original file, check if its corresponding extracted file has the same hash
ALL_MATCH=1
@ -101,10 +101,10 @@ run_basic_test() {
FILENAME=$(basename "$SOURCE_FILE")
# Find the corresponding hash from original file
ORIG_HASH=$(grep "$SOURCE_FILE" /test/complex-original-md5.txt | awk '{print $1}')
ORIG_HASH=$(grep "$SOURCE_FILE" /test-tmp/complex-original-md5.txt | awk '{print $1}')
# Find the corresponding file in the extracted directory and get its hash
EXTRACTED_FILE=$(find /test/complex-extracted -name "$FILENAME" | head -1)
EXTRACTED_FILE=$(find /test-tmp/complex-extracted -name "$FILENAME" | head -1)
if [ -z "$EXTRACTED_FILE" ]; then
echo "ERROR: File $FILENAME not found in extracted directory"
@ -112,7 +112,7 @@ run_basic_test() {
continue
fi
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test/complex-extracted-md5.txt | awk '{print $1}')
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test-tmp/complex-extracted-md5.txt | awk '{print $1}')
if [ "$ORIG_HASH" != "$EXTRACTED_HASH" ]; then
echo "ERROR: Hash mismatch for $FILENAME: original=$ORIG_HASH extracted=$EXTRACTED_HASH"
@ -135,66 +135,66 @@ run_tar_comparison_test() {
echo "=== RUNNING TAR COMPARISON TEST ==="
# Clean up test directories
rm -rf /test/standard-test /test/standard-extracted /test/reference-extracted
rm -f /test/original-checksums.txt /test/standard-checksums.txt /test/reference-checksums.txt
rm -rf /test-tmp/standard-test /test-tmp/standard-extracted /test-tmp/reference-extracted
rm -f /test-tmp/original-checksums.txt /test-tmp/standard-checksums.txt /test-tmp/reference-checksums.txt
# Create a diverse test directory structure
mkdir -p /test/standard-test/config/settings
mkdir -p /test/standard-test/data/user/documents
mkdir -p /test/standard-test/data/user/pictures
mkdir -p /test/standard-test/logs
mkdir -p /test-tmp/standard-test/config/settings
mkdir -p /test-tmp/standard-test/data/user/documents
mkdir -p /test-tmp/standard-test/data/user/pictures
mkdir -p /test-tmp/standard-test/logs
# Create various file types
echo '{"app": "tarballer", "version": "1.0"}' > /test/standard-test/config/settings/app.json
echo 'debug=true' > /test/standard-test/config/settings/debug.conf
dd if=/dev/urandom bs=1K count=10 of=/test/standard-test/data/user/documents/binary.dat 2>/dev/null
echo 'Test log entry 1' > /test/standard-test/logs/app.log
echo 'Test log entry 2' >> /test/standard-test/logs/app.log
echo '{"app": "tarballer", "version": "1.0"}' > /test-tmp/standard-test/config/settings/app.json
echo 'debug=true' > /test-tmp/standard-test/config/settings/debug.conf
dd if=/dev/urandom bs=1K count=10 of=/test-tmp/standard-test/data/user/documents/binary.dat 2>/dev/null
echo 'Test log entry 1' > /test-tmp/standard-test/logs/app.log
echo 'Test log entry 2' >> /test-tmp/standard-test/logs/app.log
# Create symlinks
ln -s ../config/settings/app.json /test/standard-test/data/config-link.json
ln -s ../../logs/app.log /test/standard-test/data/user/log-link.txt
ln -s ../config/settings/app.json /test-tmp/standard-test/data/config-link.json
ln -s ../../logs/app.log /test-tmp/standard-test/data/user/log-link.txt
# Store MD5 hashes of original files for comparison
find /test/standard-test -type f | sort | xargs md5sum > /test/original-checksums.txt
find /test-tmp/standard-test -type f | sort | xargs md5sum > /test-tmp/original-checksums.txt
# Create tarball using our utility
/bin/tarballer -source /test/standard-test -output /workdir/standard.tar.gz -prefix app
/bin/tarballer -source /test-tmp/standard-test -output /workdir/standard.tar.gz -prefix app
# Create a reference tarball using standard tar for comparison
tar -czf /workdir/reference.tar.gz -C /test/standard-test --transform 's,^./,app/,' .
tar -czf /workdir/reference.tar.gz -C /test-tmp/standard-test --transform 's,^./,app/,' .
echo '=== TARBALLER OUTPUT ==='
mkdir -p /test/standard-extracted
tar -xzf /workdir/standard.tar.gz -C /test/standard-extracted
find /test/standard-extracted -type f -o -type l | sort
mkdir -p /test-tmp/standard-extracted
tar -xzf /workdir/standard.tar.gz -C /test-tmp/standard-extracted
find /test-tmp/standard-extracted -type f -o -type l | sort
# Extract reference tarball
echo '=== REFERENCE TAR OUTPUT ==='
mkdir -p /test/reference-extracted
tar -xzf /workdir/reference.tar.gz -C /test/reference-extracted
find /test/reference-extracted -type f -o -type l | sort
mkdir -p /test-tmp/reference-extracted
tar -xzf /workdir/reference.tar.gz -C /test-tmp/reference-extracted
find /test-tmp/reference-extracted -type f -o -type l | sort
# Verify checksums match for all extracted files
echo '=== CHECKSUMS OF EXTRACTED FILES ==='
find /test/standard-extracted -type f | sort | xargs md5sum > /test/standard-checksums.txt
find /test/reference-extracted -type f | sort | xargs md5sum > /test/reference-checksums.txt
find /test-tmp/standard-extracted -type f | sort | xargs md5sum > /test-tmp/standard-checksums.txt
find /test-tmp/reference-extracted -type f | sort | xargs md5sum > /test-tmp/reference-checksums.txt
echo 'ORIGINAL FILE CHECKSUMS:'
cat /test/original-checksums.txt
cat /test-tmp/original-checksums.txt
echo 'TARBALLER EXTRACTED CHECKSUMS:'
cat /test/standard-checksums.txt
cat /test-tmp/standard-checksums.txt
echo 'REFERENCE TAR EXTRACTED CHECKSUMS:'
cat /test/reference-checksums.txt
cat /test-tmp/reference-checksums.txt
# Compare MD5 checksums systematically
echo '=== SYSTEMATIC MD5 COMPARISON ==='
# Compare original files to tarballer extraction
MATCH_COUNT=0
EXPECTED_COUNT=$(cat /test/original-checksums.txt | wc -l)
EXPECTED_COUNT=$(cat /test-tmp/original-checksums.txt | wc -l)
# For each original file, find its corresponding extracted file and compare hashes
while read -r line; do
@ -203,11 +203,11 @@ run_tar_comparison_test() {
FILENAME=$(basename "$ORIG_FILE")
# Find the corresponding file in the extracted directory
EXTRACTED_FILE=$(find /test/standard-extracted -name "$FILENAME" | grep -v ".md5-manifest.txt" | head -1)
EXTRACTED_FILE=$(find /test-tmp/standard-extracted -name "$FILENAME" | grep -v ".md5-manifest.txt" | head -1)
if [ -n "$EXTRACTED_FILE" ]; then
# Get the hash of the extracted file
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test/standard-checksums.txt | awk '{print $1}')
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test-tmp/standard-checksums.txt | awk '{print $1}')
if [ "$ORIG_HASH" = "$EXTRACTED_HASH" ]; then
MATCH_COUNT=$((MATCH_COUNT + 1))
@ -215,7 +215,7 @@ run_tar_comparison_test() {
echo "Hash mismatch for $FILENAME: original=$ORIG_HASH extracted=$EXTRACTED_HASH"
fi
fi
done < /test/original-checksums.txt
done < /test-tmp/original-checksums.txt
if [ "$MATCH_COUNT" -eq "$EXPECTED_COUNT" ]; then
echo 'SUCCESS: Tarballer extraction hashes match original files!'
@ -233,11 +233,11 @@ run_tar_comparison_test() {
FILENAME=$(basename "$ORIG_FILE")
# Find the corresponding file in the extracted directory
EXTRACTED_FILE=$(find /test/reference-extracted -name "$FILENAME" | head -1)
EXTRACTED_FILE=$(find /test-tmp/reference-extracted -name "$FILENAME" | head -1)
if [ -n "$EXTRACTED_FILE" ]; then
# Get the hash of the extracted file
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test/reference-checksums.txt | awk '{print $1}')
EXTRACTED_HASH=$(grep "$EXTRACTED_FILE" /test-tmp/reference-checksums.txt | awk '{print $1}')
if [ "$ORIG_HASH" = "$EXTRACTED_HASH" ]; then
MATCH_COUNT=$((MATCH_COUNT + 1))
@ -245,7 +245,7 @@ run_tar_comparison_test() {
echo "Hash mismatch for $FILENAME: original=$ORIG_HASH reference=$EXTRACTED_HASH"
fi
fi
done < /test/original-checksums.txt
done < /test-tmp/original-checksums.txt
if [ "$MATCH_COUNT" -eq "$EXPECTED_COUNT" ]; then
echo 'SUCCESS: Reference tar extraction hashes match original files!'
@ -256,21 +256,21 @@ run_tar_comparison_test() {
echo '=== VERIFYING SYMLINKS ==='
echo 'ORIGINAL SYMLINKS:'
find /test/standard-test -type l -exec ls -la {} \;
find /test-tmp/standard-test -type l -exec ls -la {} \;
echo 'EXTRACTED SYMLINKS:'
find /test/standard-extracted -type l -exec ls -la {} \;
find /test-tmp/standard-extracted -type l -exec ls -la {} \;
# Compare file counts to ensure all files were extracted
echo '=== FILE COUNT COMPARISON ==='
echo -n 'Original files: ' && find /test/standard-test -type f | wc -l
echo -n 'Extracted files: ' && find /test/standard-extracted -type f | wc -l
echo -n 'Original files: ' && find /test-tmp/standard-test -type f | wc -l
echo -n 'Extracted files: ' && find /test-tmp/standard-extracted -type f | wc -l
# Test symlink functionality
echo '=== TESTING SYMLINK CONTENT ==='
echo 'Original linked content:'
cat /test/standard-test/data/config-link.json
cat /test-tmp/standard-test/data/config-link.json
echo 'Extracted linked content:'
cat /test/standard-extracted/app/data/config-link.json
cat /test-tmp/standard-extracted/app/data/config-link.json
echo 'Tar comparison test completed successfully!'
return 0