✨ — Add support for Swift object storage

- Swift support
- script to upload the existing files to Swift

.gitlab-ci.yml

@@ -58,6 +58,8 @@ variables:
   services:
     - name: rroemhild/test-openldap
       alias: rroemhild-test-openldap
+    - name: swiftstack/picoswiftstack
+      alias: swiftstack-picoswiftstack
 .pg_template: &pg_definition
   <<: *tests_template
   needs:
@@ -67,6 +69,8 @@ variables:
       alias: postgres
     - name: rroemhild/test-openldap
       alias: rroemhild-test-openldap
+    - name: swiftstack/picoswiftstack
+      alias: swiftstack-picoswiftstack
 .mysql_template: &mysql_definition
   <<: *tests_template
   needs:
@@ -76,6 +80,8 @@ variables:
       alias: mariadb
     - name: rroemhild/test-openldap
       alias: rroemhild-test-openldap
+    - name: swiftstack/picoswiftstack
+      alias: swiftstack-picoswiftstack
 
 ### Publish tag changelog and create a toot
 ##

CHANGELOG

@@ -5,6 +5,7 @@ Revision history for Lufi
 	- Use Weblate instead of Zanata for translations (https://weblate.framasoft.org/projects/lufi/development/)
 	- Add config API endpoint (#183)
 	- Show latest tag and commit of the instance in about page and config API endpoint (#174)
+	- Add support for Swift object storage
 
 0.04.6 2019-11-07
 	- Now can send large files (>2Gio) while using a DB other than SQLite (#165)

Makefile

@@ -30,6 +30,14 @@ ldap:
 
 ldapdev: ldap dev
 
+swift:
+	sudo docker run -d --rm -p 8080:8080 --hostname="picoswiftstack" --name="picoswiftstack" swiftstack/picoswiftstack; exit 0
+	@echo "Sleeping 20 seconds to let picoswiftstack start"
+	@sleep 20
+	sudo docker exec picoswiftstack get_auth
+
+swiftdev: swift dev
+
 devlog:
 	multitail log/development.log
 

cpanfile

@@ -6,9 +6,9 @@ requires 'Mojolicious::Plugin::Mail';
 requires 'Mojolicious::Plugin::GzipStatic';
 requires 'Mojolicious::Plugin::StaticCache';
 requires 'Mojolicious::Plugin::CSPHeader', '>= 0.06';
-requires 'Mojolicious::Plugin::FiatTux::Helpers', '== 0.10', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-helpers/-/archive/0.10/mojolicious-plugin-fiattux-helpers-0.10.tar.gz';
+requires 'Mojolicious::Plugin::FiatTux::Helpers', '== 0.12', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-helpers/-/archive/0.12/mojolicious-plugin-fiattux-helpers-0.12.tar.gz';
-requires 'Mojolicious::Plugin::FiatTux::GrantAccess', '== 0.06', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-grantaccess/-/archive/0.06/mojolicious-plugin-fiattux-grantaccess-0.06.tar.gz';
+requires 'Mojolicious::Plugin::FiatTux::GrantAccess', '== 0.07', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-grantaccess/-/archive/0.07/mojolicious-plugin-fiattux-grantaccess-0.07.tar.gz';
-requires 'Mojolicious::Plugin::FiatTux::Themes', '== 0.02', url => 'https://framagit.org/fiat-tux/mojolicious/mojolicious-plugin-fiattux-themes/-/archive/0.02/mojolicious-plugin-fiattux-themes-0.02.tar.gz';
+requires 'Mojolicious::Plugin::FiatTux::Themes', '== 0.02', url => 'https://framagit.org/fiat-tux/mojolicious/fiat-tux/mojolicious-plugin-fiattux-themes/-/archive/0.02/mojolicious-plugin-fiattux-themes-0.02.tar.gz';
 requires 'Filesys::DiskUsage';
 requires 'Switch';
 requires 'Locale::Maketext';
@@ -55,3 +55,9 @@ feature 'mysql', 'MySQL support' => sub {
     requires 'Mojo::mysql';
     requires 'Mojolicious::Plugin::PgURLHelper';
 };
+feature 'swift-storage', 'Openstack Swift object storage support' => sub {
+    requires 'Net::OpenStack::Swift';
+};
+feature 'test' => sub {
+    requires 'Devel::Cover';
+};

lib/Lufi.pm

@@ -66,6 +66,11 @@ sub startup {
     # Helpers
     $self->plugin('Lufi::Plugin::Helpers');
 
+    # Now helpers has been loaded, time to check Swift container
+    if ($config->{swift}) {
+        $self->check_swift_container();
+    }
+
     # Recurrent task
     Mojo::IOLoop->recurring(2 => sub {
         my $loop = shift;
@@ -74,8 +79,10 @@ sub startup {
     });
 
     # Create directory if needed
-    mkdir($self->config('upload_dir'), 0700) unless (-d $self->config('upload_dir'));
+    if (!defined($config->{swift})) {
-    die ('The upload directory ('.$self->config('upload_dir').') is not writable') unless (-w $self->config('upload_dir'));
+        mkdir($self->config('upload_dir'), 0700) unless (-d $self->config('upload_dir'));
+        die ('The upload directory ('.$self->config('upload_dir').') is not writable') unless (-w $self->config('upload_dir'));
+    }
 
     # Configure sessions
     my $sessions = Mojolicious::Sessions->new;

lib/Lufi/Command/copyFilesToSwift.pm

@@ -0,0 +1,68 @@
+package Lufi::Command::copyFilesToSwift;
+use Mojo::Base 'Mojolicious::Command';
+use File::Spec;
+use Term::ProgressBar;
+
+has description => 'Copy files from filesystem to Swift object storage';
+has usage       => sub { shift->extract_usage };
+
+sub run {
+    my $c = shift;
+
+    if ($c->app->config('swift')) {
+        $c->app->check_swift_container();
+        my @dirs = glob(File::Spec->catdir($c->app->config('upload_dir'), '*'));
+
+        say sprintf('%d folders to upload to Swift (can\'t say how many files, or the total size, sorry). This can take some time.', scalar(@dirs));
+        print 'Do you want to continue? [Y/n] ';
+        my $confirm = <STDIN>;
+
+        if ($confirm =~ m/yes|y/i) {
+            my $progress = Term::ProgressBar->new({ count => scalar(@dirs), ETA => 'linear', name => 'Copying to Swift'});
+            for my $dir (@dirs) {
+                my @files = glob(File::Spec->catfile($dir, '*'));
+                for my $file (@files) {
+                    my ($volume, $directories, $filename) = File::Spec->splitpath($file);
+                    my @file_dirs = File::Spec->splitdir($directories);
+                    my $short     = ($file_dirs[-1] ne '') ? $file_dirs[-1] : $file_dirs[-2];
+                    my $path      = File::Spec->catfile($short, $filename);
+
+                    open my $fh, '<', $file or die sprintf('Unable to open file %s: %s', $file, $!);
+
+                    $c->app->swift->put_object(
+                        container_name => $c->app->config('swift')->{container},
+                        object_name    => $path,
+                        content_length => -s $file,
+                        content        => $fh
+                    );
+                    close $fh;
+                }
+                $progress->update();
+            }
+            say sprintf('The copy to Swift object storage has ended. You can test Lufi, then delete `%s` directory', $c->app->config('upload_dir'));
+        } else {
+            say 'You want to stop. No problem.';
+        }
+    } else {
+        say 'You didn\'t configure `swift` in your config file. Exiting.';
+        exit 1;
+    }
+}
+
+=encoding utf8
+
+=head1 NAME
+
+Lufi::Command::copyFilesToSwift Copy files from filesystem to Swift object storage
+
+=head1 SYNOPSIS
+
+  Usage: script/lufi copyFilesToSwift
+
+  This command needs you to:
+    - set `upload_dir` in your config file (otherwise, it will use the default path, `files` to copy files from)
+    - configure `swift` with correct informations in your config file
+
+=cut
+
+1;

lib/Lufi/Command/sqliteToOtherDB.pm

@@ -1,5 +1,6 @@
 package Lufi::Command::sqliteToOtherDB;
 use Mojo::Base 'Mojolicious::Command';
+use Lufi::DB::BreakingChange;
 use Lufi::DB::File;
 use Lufi::DB::Slice;
 use Lufi::DB::Invitation;
@@ -9,7 +10,7 @@ use Term::ProgressBar;
 use Lufi::DefaultConfig qw($default_config);
 
 has description => 'Migrate the records from a SQLite db to the currently configured database';
-has usage => sub { shift->extract_usage };
+has usage       => sub { shift->extract_usage };
 
 sub run {
     my $c = shift;
@@ -36,8 +37,9 @@ sub run {
     my $files       = $sqlite->db->select('files', undef)->hashes;
     my $slices      = $sqlite->db->select('slices', undef)->hashes;
     my $invitations = $sqlite->db->select('invitations', undef)->hashes;
+    my $changes     = $sqlite->db->select('breakingchanges', undef)->hashes;
 
-    my $progress = Term::ProgressBar->new({count => $files->size + $slices->size + $invitations->size});
+    my $progress = Term::ProgressBar->new({count => $files->size + $slices->size + $invitations->size + $changes->size});
 
     $files->each(sub {
         my ($file, $num) = @_;
@@ -69,7 +71,6 @@ sub run {
         Lufi::DB::Slice->new(app => $c->app)
                       ->short($slice->{short})
                       ->j($slice->{j})
-                      ->path($slice->{path})
                       ->write();
 
         $progress->update();
@@ -92,13 +93,22 @@ sub run {
                             ->write();
         $progress->update();
     });
+    $changes->each(sub {
+        my ($change, $num) = @_;
+
+        Lufi::DB::BreakingChange->new(app => $c->app)
+                                ->change($change->{change})
+                                ->ack($change->{ack})
+                                ->write();
+        $progress->update();
+    });
 }
 
 =encoding utf8
 
 =head1 NAME
 
-Lufi::Command::cron::sqliteToOtherDB Migrate the records from a SQLite db to the currently configured database
+Lufi::Command::sqliteToOtherDB Migrate the records from a SQLite db to the currently configured database
 
 =head1 SYNOPSIS
 

lib/Lufi/Command/theme.pm

@@ -2,7 +2,7 @@
 package Lufi::Command::theme;
 use Mojo::Base 'Mojolicious::Commands';
 use FindBin qw($Bin);
-use File::Spec qw(catfile cat dir);
+use File::Spec qw(catfile catdir);
 use File::Path qw(make_path);
 
 has description => 'Create new theme skeleton.';

lib/Lufi/Controller/Files.pm

@@ -100,7 +100,8 @@ sub upload {
                     }
                 }
                 # Check that we have enough space (multiplying by 2 since it's encrypted, it takes more place that the original file)
-                if ($json->{part} == 0 && ($json->{size} * 2) >= dfportable($c->config('upload_dir'))->{bavail}) {
+                # Only check if using filesystem, not Swift storage
+                if (!defined($c->config('swift')) && $json->{part} == 0 && ($json->{size} * 2) >= dfportable($c->config('upload_dir'))->{bavail}) {
                     $stop = 1;
                     return $ws->send(decode('UTF-8', encode_json(
                         {
@@ -190,19 +191,12 @@ sub upload {
                         # If we already have a part, it's a resend because the websocket has been broken
                         # In this case, we don't need to rewrite the file
                         unless ($f->slices->grep(sub { $_->j == $json->{part} })->size) {
-                            # Create directory
-                            my $dir = catdir($c->config('upload_dir'), $f->short);
-                            mkdir($dir, 0700) unless (-d $dir);
-
                             # Create slice file
-                            my $file = catfile($dir, $json->{part}.'.part');
                             my $s    = Lufi::DB::Slice->new(
                                 app   => $c->app,
                                 short => $f->short,
-                                j     => $json->{part},
+                                j     => $json->{part}
-                                path  => $file
+                            )->store($text);
-                            );
-                            Mojo::File->new($file)->spurt($text);
                             push @{$f->slices}, $s;
                             $s->write;
 
@@ -327,7 +321,7 @@ sub download {
 
                             # Get the slice
                             my $e    = $f->slices->[$num];
-                            my $text = Mojo::File->new($e->path)->slurp;
+                            my $text = $e->retrieve();
 
                             my ($json2) = split('XXMOJOXX', $text, 2);
                             $json2 = decode 'UTF-8', $json2;

lib/Lufi/DB/File.pm

@@ -148,13 +148,37 @@ sub delete {
 
     $c->slices->each(sub {
         my ($e, $num) = @_;
-        unlink $e->path;
+        $e->delete_file();
     });
-    rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
+    $c->delete_path
-    $c->deleted(1);
+      ->deleted(1)
+      ->write;
 
-    $c->write;
+    return $c;
+}
 
+=head2 delete_path
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>delete_path()>
+
+=item B<Arguments> : none
+
+=item B<Purpose>   : delete the directory of the slices on filesystem or Swift object storage
+
+=item B<Returns>   : the db accessor object
+
+=back
+
+=cut
+
+sub delete_path {
+    my $c = shift;
+
+    if (!defined($c->app->config('swift'))) {
+        rmdir Mojo::File->new($c->app->config('upload_dir'), $c->short);
+    }
     return $c;
 }
 

lib/Lufi/DB/Slice.pm

@@ -1,14 +1,17 @@
 # vim:set sw=4 ts=4 sts=4 ft=perl expandtab:
 package Lufi::DB::Slice;
 use Mojo::Base -base;
+use Encode 'encode';
+use File::Spec::Functions;
 use Mojo::Collection 'c';
 
 has 'short';
 has 'j';
-has 'path';
 has 'record' => 0;
 has 'app';
 
+=encoding utf8
+
 =head1 NAME
 
 Lufi::DB::Slice - DB abstraction layer for Lufi file
@@ -28,8 +31,6 @@ Have a look at Lufi::DB::Slice::SQLite's code: it's simple and may be more under
 
 =item B<j>     : integer
 
-=item B<path>  : string
-
 =item B<app>   : A mojolicious object
 
 =back
@@ -96,15 +97,121 @@ sub write {
     my $c = shift;
 
     if ($c->record) {
-        $c->app->dbi->db->query('UPDATE slices SET short = ?, j = ?, path = ? WHERE short = ? AND j = ?', $c->short, $c->j, $c->path, $c->short, $c->j);
+        $c->app->dbi->db->query('UPDATE slices SET short = ?, j = ? WHERE short = ? AND j = ?', $c->short, $c->j, $c->short, $c->j);
     } else {
-        $c->app->dbi->db->query('INSERT INTO slices (short, j, path) VALUES (?, ?, ?)', $c->short, $c->j, $c->path);
+        $c->app->dbi->db->query('INSERT INTO slices (short, j) VALUES (?, ?)', $c->short, $c->j);
         $c->record(1);
     }
 
     return $c;
 }
 
+=head2 store
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>store($text)>
+
+=item B<Arguments> : a scalar value
+
+=item B<Purpose>   : will store the content to the object's path, either on filesystem or on Swift object storage
+
+=item B<Returns>   : the db accessor object
+
+=back
+
+=cut
+
+sub store {
+    my $c    = shift;
+    my $text = shift;
+
+    if ($c->app->config('swift')) {
+        $c->app->swift->put_object(
+            container_name => $c->app->config('swift')->{container},
+            object_name    => $c->get_path(),
+            content_length => length(Encode::encode_utf8($text)),
+            content        => $text
+        );
+    } else {
+        # Create directory
+        my $dir = catfile($c->app->config('upload_dir'), $c->short);
+        mkdir($dir, 0700) unless (-d $dir);
+
+        # Write file
+        my $file = catfile($c->app->config('upload_dir'), $c->get_path());
+        Mojo::File->new($file)->spurt($text);
+    }
+
+    return $c;
+}
+
+=head2 retrieve
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>retrieve>
+
+=item B<Arguments> : none
+
+=item B<Purpose>   : get file from storage, either filesystem or Swift object storage
+
+=item B<Returns>   : the data from the file
+
+=back
+
+=cut
+
+sub retrieve {
+    my $c      = shift;
+    my $upload = shift;
+
+    if ($c->app->config('swift')) {
+        my $file;
+        $c->app->swift->get_object(
+            container_name => $c->app->config('swift')->{container},
+            object_name    => $c->get_path(),
+            write_code => sub {
+                my ($status, $message, $headers, $chunk) = @_;
+                $file .= $chunk;
+            }
+        );
+        return $file;
+    } else {
+        my $file = catfile($c->app->config('upload_dir'), $c->get_path());
+        return Mojo::File->new($file)->slurp;
+    }
+}
+=head2 delete_file
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>delete_file()>
+
+=item B<Arguments> : none
+
+=item B<Purpose>   : delete the file on filesystem or Swift object storage
+
+=item B<Returns>   : the db accessor object
+
+=back
+
+=cut
+
+sub delete_file {
+    my $c   = shift;
+
+    if ($c->app->config('swift')) {
+        $c->app->swift->delete_object({
+            container_name => $c->app->config('swift')->{container},
+            object_name    => $c->get_path()
+        });
+    } else {
+        my $file = catfile($c->app->config('upload_dir'), $c->get_path());
+        unlink $file or warn sprintf('Could not unlink %s: %s', $file, $!);
+    }
+    return $c;
+}
 =head2 get_slices_of_file
 
 =over 1
@@ -147,7 +254,7 @@ sub get_slices_of_file {
 
 =item B<Arguments> : none
 
-=item B<Purpose>   : delete all file records from database unconditionnally
+=item B<Purpose>   : delete all slices records from database unconditionnally
 
 =item B<Returns>   : nothing
 
@@ -161,6 +268,50 @@ sub delete_all {
     $c->app->dbi->db->delete('slices');
 }
 
+=head2 path
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>path()>
+
+=item B<Arguments> : non
+
+=item B<Purpose>   : format the path of the file, relative to the directory of the Swift object storage
+
+=item B<Returns>   : the path of the file
+
+=back
+
+=cut
+
+sub get_path {
+    my $c        = shift;
+
+    return catfile($c->short, sprintf('%d.part', $c->j));
+}
+
+=head2 count
+
+=over 1
+
+=item B<Usage>     : C<$c-E<gt>count()>
+
+=item B<Arguments> : none
+
+=item B<Purpose>   : get count of slices records from database
+
+=item B<Returns>   : integer
+
+=back
+
+=cut
+
+sub count {
+    my $c = shift;
+
+    return $c->app->dbi->db->query('SELECT count(*) AS count FROM slices')->hashes->first->{count};
+}
+
 =head2 _slurp
 
 =over 1
@@ -195,7 +346,6 @@ sub _slurp {
     if ($slice) {
         $c->short($slice->{short});
         $c->j($slice->{j});
-        $c->path($slice->{path});
 
         $c->record(1);
     }

lib/Lufi/DB/Slice/Mysql.pm

@@ -7,6 +7,8 @@ sub new {
 
     $c = $c->SUPER::new(@_);
 
+    $c = $c->_slurp if defined $c->record;
+
     return $c;
 }
 

lib/Lufi/DB/Slice/Pg.pm

@@ -7,6 +7,8 @@ sub new {
 
     $c = $c->SUPER::new(@_);
 
+    $c = $c->_slurp if defined $c->record;
+
     return $c;
 }
 

lib/Mounter.pm

@@ -27,6 +27,9 @@ sub startup {
     # Compress static assets
     $self->plugin('GzipStatic');
 
+    # Fiat Tux helpers
+    $self->plugin('FiatTux::Helpers');
+
     # Headers
     $self->plugin('Lufi::Plugin::Headers');
 

lufi.conf.template

@@ -99,14 +99,25 @@
     # Array of authorized domains for API calls.
     # If you want to authorize everyone to use the API: ['*']
     # optional, no domains allowed by default
-    #allowed_domains   => ['http://1.example.com', 'http://2.example.com'],
+    #allowed_domains => ['http://1.example.com', 'http://2.example.com'],
 
     # Define a path to the upload directory, where the uploaded files will be stored
     # You can define it relative to lufi directory or set an absolute path
     # Remember that it has to be in a directory writable by Lufi user
-    # DO NOT CHANGE THIS IF FILES HAVE BEEN ALREADY UPLOADED: THEY WILL NOT BE DOWNLOADABLE ANYMORE
     # optional, default is 'files'
-    #upload_dir           => 'files',
+    #upload_dir => 'files',
+
+    # You can store files on Swift object storage (https://en.wikipedia.org/wiki/OpenStack#Swift) instead of filesystem
+    # Please read https://metacpan.org/pod/Net::OpenStack::Swift#SYNOPSIS to know how to configure this setting
+    # IMPORTANT: add a `container` key in it, to let Lufi know which container to use. This is not a regular Net::OpenStack::Swift setting, but Lufi need it.
+    # optional, no default
+    #swift => {
+    #  auth_url    => 'https://auth-endpoint-url/v2.0',
+    #  user        => 'userid',
+    #  password    => 'password',
+    #  tenant_name => 'project_id',
+    #  container   => 'lufi'
+    #},
 
     # Allow to add a password on files, asked before allowing to download files
     # optional, default is 0

t/mysql.conf

@@ -32,6 +32,10 @@
     # optional, default is ['fdjsofjoihrei'], PLEASE, CHANGE IT
     #secrets        => ['fdjsofjoihrei'],
 
+    # Name of the instance, displayed next to the logo
+    # optional, default is Lufi
+    #instance_name => 'Lufi',
+
     # Choose a theme. See the available themes in `themes` directory
     # Optional, default is 'default'
     #theme         => 'default',
@@ -97,10 +101,6 @@
     # optional, no domains allowed by default
     #allowed_domains   => ['http://1.example.com', 'http://2.example.com'],
 
-    # If set, the shortened URLs will use this domain
-    # optional
-    #fixed_domain => 'example.org',
-
     # Define a path to the upload directory, where the uploaded files will be stored
     # You can define it relative to lufi directory or set an absolute path
     # Remember that it has to be in a directory writable by Lufi user
@@ -108,6 +108,12 @@
     # optional, default is 'files'
     #upload_dir           => 'files',
 
+    # You can store files on Swift object storage (https://en.wikipedia.org/wiki/OpenStack#Swift) instead of filesystem
+    # Please read https://metacpan.org/pod/Net::OpenStack::Swift#SYNOPSIS to know how to configure this setting
+    # IMPORTANT: add a `container` key in it, to let Lufi know which container to use. This is not a regular Net::OpenStack::Swift setting, but Lufi need it.
+    # optional, no default
+    #swift => { auth_url => 'http://swiftstack-picoswiftstack:8080/auth/v1.0', user => 'test', password => 'test', container => 'lufi', auth_version => '1.0' },
+
     # Allow to add a password on files, asked before allowing to download files
     # optional, default is 0
     allow_pwd_on_files => 1,
@@ -228,11 +234,47 @@
     #
     # Define the attributes like this: `lufi_attribute_name => 'LDAP_attribute_name'`
     # Note that you can’t use `username` as a Lufi attribute name: this name is reserved and will contain the login of the user
+    # optional, no default
     #ldap_map_attr => {
     #    displayname => 'cn',
     #    mail        => 'mail'
     #},
 
+    # When using LDAP authentication, LDAP users can invite people (by mail) to use Lufi to send them files without
+    # being authenticated.
+    # This is where you configure the behavior of the invitations.
+    # You may need to fetch some attributes from LDAP to use some invitations settings. See `ldap_map_attr` above.
+    # optional, no default
+    #invitations => {
+    #   # The name of the key set in `ldap_map_attr` (above) that corresponds to the mail of the LDAP user
+    #   # optional, default is `mail`
+    #   mail_attr => 'mail',
+    #   # The `From` header of invitation mail can be the mail of the LDAP user
+    #   # Be sure to have a mail system that will correctly send the mail from your users! (DKIM, SPF…)
+    #   # To enable this feature, set it to 1
+    #   # optional, disabled by default
+    #   send_invitation_with_ldap_user_mail => 1,
+    #   # The user is able to set an expiration delay for the invitation.
+    #   # This expiration delay can’t be more than this setting (in days).
+    #   # optional, default is 30 days
+    #   max_invitation_expiration_delay => 30,
+    #   # Once the guest has submitted his files, he has an additional period of time to submit forgotten files.
+    #   # You can set that additional period of time in minutes here.
+    #   # To disable that feature, set it to 0 or less
+    #   # optional, default is 10 minutes
+    #   max_additional_period => 10,
+    #   # Lufi follows privacy-by-design, so, by default, no files URLs (with the decode secret) are stored in database.
+    #   # However, the concern is different for this case. Storing files URLs makes users able to retrieve the guests’ sent files
+    #   # from their `invitations` page.
+    #   # Set to 1 to store guests’ files URLs in database
+    #   # optional, default is 0 (disabled)
+    #   save_files_url_in_db => 0,
+    #   # Users can resend the invitation to their guest. This does not extend the invitation’s expiration delay unless you
+    #   # set this option to 1.
+    #   # optional, default is 0 (disabled)
+    #   extend_invitation_expiration_on_resend => 0,
+    #},
+
     #########################
     # Htpasswd authentication
     #########################

t/postgresql.conf

@@ -32,6 +32,10 @@
     # optional, default is ['fdjsofjoihrei'], PLEASE, CHANGE IT
     #secrets        => ['fdjsofjoihrei'],
 
+    # Name of the instance, displayed next to the logo
+    # optional, default is Lufi
+    #instance_name => 'Lufi',
+
     # Choose a theme. See the available themes in `themes` directory
     # Optional, default is 'default'
     #theme         => 'default',
@@ -97,10 +101,6 @@
     # optional, no domains allowed by default
     #allowed_domains   => ['http://1.example.com', 'http://2.example.com'],
 
-    # If set, the shortened URLs will use this domain
-    # optional
-    #fixed_domain => 'example.org',
-
     # Define a path to the upload directory, where the uploaded files will be stored
     # You can define it relative to lufi directory or set an absolute path
     # Remember that it has to be in a directory writable by Lufi user
@@ -108,6 +108,12 @@
     # optional, default is 'files'
     #upload_dir           => 'files',
 
+    # You can store files on Swift object storage (https://en.wikipedia.org/wiki/OpenStack#Swift) instead of filesystem
+    # Please read https://metacpan.org/pod/Net::OpenStack::Swift#SYNOPSIS to know how to configure this setting
+    # IMPORTANT: add a `container` key in it, to let Lufi know which container to use. This is not a regular Net::OpenStack::Swift setting, but Lufi need it.
+    # optional, no default
+    #swift => { auth_url => 'http://swiftstack-picoswiftstack:8080/auth/v1.0', user => 'test', password => 'test', container => 'lufi', auth_version => '1.0' },
+
     # Allow to add a password on files, asked before allowing to download files
     # optional, default is 0
     allow_pwd_on_files => 1,
@@ -172,6 +178,7 @@
         #port     => 5432,
         user     => 'lufi',
         pwd      => 'lufi_pwd'
+    #    # https://mojolicious.org/perldoc/Mojo/Pg#max_connections
     #    # optional, default is 1
     #    #max_connections => 1,
     },
@@ -212,11 +219,47 @@
     #
     # Define the attributes like this: `lufi_attribute_name => 'LDAP_attribute_name'`
     # Note that you can’t use `username` as a Lufi attribute name: this name is reserved and will contain the login of the user
+    # optional, no default
     #ldap_map_attr => {
     #    displayname => 'cn',
     #    mail        => 'mail'
     #},
 
+    # When using LDAP authentication, LDAP users can invite people (by mail) to use Lufi to send them files without
+    # being authenticated.
+    # This is where you configure the behavior of the invitations.
+    # You may need to fetch some attributes from LDAP to use some invitations settings. See `ldap_map_attr` above.
+    # optional, no default
+    #invitations => {
+    #   # The name of the key set in `ldap_map_attr` (above) that corresponds to the mail of the LDAP user
+    #   # optional, default is `mail`
+    #   mail_attr => 'mail',
+    #   # The `From` header of invitation mail can be the mail of the LDAP user
+    #   # Be sure to have a mail system that will correctly send the mail from your users! (DKIM, SPF…)
+    #   # To enable this feature, set it to 1
+    #   # optional, disabled by default
+    #   send_invitation_with_ldap_user_mail => 1,
+    #   # The user is able to set an expiration delay for the invitation.
+    #   # This expiration delay can’t be more than this setting (in days).
+    #   # optional, default is 30 days
+    #   max_invitation_expiration_delay => 30,
+    #   # Once the guest has submitted his files, he has an additional period of time to submit forgotten files.
+    #   # You can set that additional period of time in minutes here.
+    #   # To disable that feature, set it to 0 or less
+    #   # optional, default is 10 minutes
+    #   max_additional_period => 10,
+    #   # Lufi follows privacy-by-design, so, by default, no files URLs (with the decode secret) are stored in database.
+    #   # However, the concern is different for this case. Storing files URLs makes users able to retrieve the guests’ sent files
+    #   # from their `invitations` page.
+    #   # Set to 1 to store guests’ files URLs in database
+    #   # optional, default is 0 (disabled)
+    #   save_files_url_in_db => 0,
+    #   # Users can resend the invitation to their guest. This does not extend the invitation’s expiration delay unless you
+    #   # set this option to 1.
+    #   # optional, default is 0 (disabled)
+    #   extend_invitation_expiration_on_resend => 0,
+    #},
+
     #########################
     # Htpasswd authentication
     #########################

t/sqlite.conf

@@ -32,6 +32,10 @@
     # optional, default is ['fdjsofjoihrei'], PLEASE, CHANGE IT
     #secrets        => ['fdjsofjoihrei'],
 
+    # Name of the instance, displayed next to the logo
+    # optional, default is Lufi
+    #instance_name => 'Lufi',
+
     # Choose a theme. See the available themes in `themes` directory
     # Optional, default is 'default'
     #theme         => 'default',
@@ -97,10 +101,6 @@
     # optional, no domains allowed by default
     #allowed_domains   => ['http://1.example.com', 'http://2.example.com'],
 
-    # If set, the shortened URLs will use this domain
-    # optional
-    #fixed_domain => 'example.org',
-
     # Define a path to the upload directory, where the uploaded files will be stored
     # You can define it relative to lufi directory or set an absolute path
     # Remember that it has to be in a directory writable by Lufi user
@@ -108,6 +108,12 @@
     # optional, default is 'files'
     #upload_dir           => 'files',
 
+    # You can store files on Swift object storage (https://en.wikipedia.org/wiki/OpenStack#Swift) instead of filesystem
+    # Please read https://metacpan.org/pod/Net::OpenStack::Swift#SYNOPSIS to know how to configure this setting
+    # IMPORTANT: add a `container` key in it, to let Lufi know which container to use. This is not a regular Net::OpenStack::Swift setting, but Lufi need it.
+    # optional, no default
+    #swift => { auth_url => 'http://swiftstack-picoswiftstack:8080/auth/v1.0', user => 'test', password => 'test', container => 'lufi', auth_version => '1.0' },
+
     # Allow to add a password on files, asked before allowing to download files
     # optional, default is 0
     allow_pwd_on_files => 1,
@@ -217,11 +223,47 @@
     #
     # Define the attributes like this: `lufi_attribute_name => 'LDAP_attribute_name'`
     # Note that you can’t use `username` as a Lufi attribute name: this name is reserved and will contain the login of the user
+    # optional, no default
     #ldap_map_attr => {
     #    displayname => 'cn',
     #    mail        => 'mail'
     #},
 
+    # When using LDAP authentication, LDAP users can invite people (by mail) to use Lufi to send them files without
+    # being authenticated.
+    # This is where you configure the behavior of the invitations.
+    # You may need to fetch some attributes from LDAP to use some invitations settings. See `ldap_map_attr` above.
+    # optional, no default
+    #invitations => {
+    #   # The name of the key set in `ldap_map_attr` (above) that corresponds to the mail of the LDAP user
+    #   # optional, default is `mail`
+    #   mail_attr => 'mail',
+    #   # The `From` header of invitation mail can be the mail of the LDAP user
+    #   # Be sure to have a mail system that will correctly send the mail from your users! (DKIM, SPF…)
+    #   # To enable this feature, set it to 1
+    #   # optional, disabled by default
+    #   send_invitation_with_ldap_user_mail => 1,
+    #   # The user is able to set an expiration delay for the invitation.
+    #   # This expiration delay can’t be more than this setting (in days).
+    #   # optional, default is 30 days
+    #   max_invitation_expiration_delay => 30,
+    #   # Once the guest has submitted his files, he has an additional period of time to submit forgotten files.
+    #   # You can set that additional period of time in minutes here.
+    #   # To disable that feature, set it to 0 or less
+    #   # optional, default is 10 minutes
+    #   max_additional_period => 10,
+    #   # Lufi follows privacy-by-design, so, by default, no files URLs (with the decode secret) are stored in database.
+    #   # However, the concern is different for this case. Storing files URLs makes users able to retrieve the guests’ sent files
+    #   # from their `invitations` page.
+    #   # Set to 1 to store guests’ files URLs in database
+    #   # optional, default is 0 (disabled)
+    #   save_files_url_in_db => 0,
+    #   # Users can resend the invitation to their guest. This does not extend the invitation’s expiration delay unless you
+    #   # set this option to 1.
+    #   # optional, default is 0 (disabled)
+    #   extend_invitation_expiration_on_resend => 0,
+    #},
+
     #########################
     # Htpasswd authentication
     #########################

t/test.t

@@ -77,6 +77,9 @@ BEGIN {
 Lufi::DB::Slice->new(app => $m)->delete_all;
 Lufi::DB::File->new(app => $m)->delete_all;
 
+$config_file = Mojo::File->new($cfile->to_abs->to_string);
+$config_orig = $config_file->slurp;
+
 my $t = Test::Mojo->new('Lufi');
 
 ## Wait for short generation
@@ -103,6 +106,12 @@ test_infos_api(true);
 auth_test_suite('zoidberg', 'zoidberg');
 restore_config();
 
+## Test Swift object storage
+switch_to_swift();
+test_upload_file();
+test_download_file();
+restore_config();
+
 done_testing();
 
 ######
@@ -263,9 +272,7 @@ sub restore_config {
 }
 
 sub switch_to_htpasswd {
-    $config_file    = Mojo::File->new($cfile->to_abs->to_string);
+    $config_content = $config_orig;
-    $config_content = $config_file->slurp;
-    $config_orig    = $config_content;
     $config_content =~ s/#?htpasswd.*/htpasswd => 't\/lufi.passwd',/gm;
     $config_file->spurt($config_content);
 
@@ -291,3 +298,17 @@ sub switch_to_ldap {
     ## Wait for short generation
     sleep 5;
 }
+
+sub switch_to_swift {
+    $config_content = $config_orig;
+    $config_content =~ s/^( +)#?swift => \{ auth_url/$1swift => { auth_url/gm;
+    $config_file->spurt($config_content);
+
+    Lufi::DB::Slice->new(app => $m)->delete_all;
+    Lufi::DB::File->new(app => $m)->delete_all;
+
+    $t = Test::Mojo->new('Lufi');
+
+    ## Wait for short generation
+    sleep 5;
+}

themes/default/lib/Lufi/I18N/en.po

@@ -138,15 +138,15 @@ msgstr "Copy all links to clipboard"
 msgid "Copy to clipboard"
 msgstr "Copy to clipboard"
 
-#: lib/Lufi/Controller/Files.pm:507
+#: lib/Lufi/Controller/Files.pm:502
 msgid "Could not delete the file. You are not authenticated."
 msgstr "Could not delete the file. You are not authenticated."
 
-#: lib/Lufi/Controller/Files.pm:489
+#: lib/Lufi/Controller/Files.pm:484
 msgid "Could not find the file. Are you sure of the URL and the token?"
 msgstr "Could not find the file. Are you sure of the URL and the token?"
 
-#: lib/Lufi/Controller/Files.pm:400
+#: lib/Lufi/Controller/Files.pm:395
 msgid "Could not find the file. Are you sure of the URL?"
 msgstr "Could not find the file. Are you sure of the URL?"
 
@@ -222,15 +222,15 @@ msgstr "Emails"
 msgid "Encrypting part XX1 of XX2"
 msgstr "Encrypting part XX1 of XX2"
 
-#: lib/Lufi/Controller/Files.pm:289
+#: lib/Lufi/Controller/Files.pm:284
 msgid "Error: the file existed but was deleted."
 msgstr "Error: the file existed but was deleted."
 
-#: lib/Lufi/Controller/Files.pm:369
+#: lib/Lufi/Controller/Files.pm:364
 msgid "Error: the file has not been sent entirely."
 msgstr "Error: the file has not been sent entirely."
 
-#: lib/Lufi/Controller/Files.pm:379
+#: lib/Lufi/Controller/Files.pm:374
 msgid "Error: unable to find the file. Are you sure of your URL?"
 msgstr "Error: unable to find the file. Are you sure of your URL?"
 
@@ -250,7 +250,7 @@ msgstr "Expires at"
 msgid "Export localStorage data"
 msgstr "Export localStorage data"
 
-#: lib/Lufi/Controller/Files.pm:471
+#: lib/Lufi/Controller/Files.pm:466
 msgid "File deleted"
 msgstr "File deleted"
 
@@ -436,7 +436,7 @@ msgid "Name of the zip file"
 msgstr "Name of the zip file"
 
 #. (format_bytes($json->{size})
-#: lib/Lufi/Controller/Files.pm:108
+#: lib/Lufi/Controller/Files.pm:109
 msgid "No enough space available on the server for this file (size: %1)."
 msgstr "No enough space available on the server for this file (size: %1)."
 
@@ -551,7 +551,7 @@ msgid "Sorry, your invitation has expired or has been deleted."
 msgstr "Sorry, your invitation has expired or has been deleted."
 
 #. ($invit->ldap_user_mail)
-#: lib/Lufi/Controller/Files.pm:122
+#: lib/Lufi/Controller/Files.pm:123
 msgid "Sorry, your invitation has expired or has been deleted. Please contact %1 to have another invitation."
 msgstr "Sorry, your invitation has expired or has been deleted. Please contact %1 to have another invitation."
 
@@ -585,7 +585,7 @@ msgstr "The email subject can't be empty."
 msgid "The expiration delay (%1) is not between 1 and %2 days."
 msgstr "The expiration delay (%1) is not between 1 and %2 days."
 
-#: lib/Lufi/Controller/Files.pm:468
+#: lib/Lufi/Controller/Files.pm:463
 msgid "The file has already been deleted"
 msgstr "The file has already been deleted"
 
@@ -640,11 +640,11 @@ msgstr "The mail has been sent."
 msgid "The original (and only for now) author is <a href=\"https://fiat-tux.fr\" class=\"classic\">Luc Didry</a>."
 msgstr "The original (and only for now) author is <a href=\"https://fiat-tux.fr\" class=\"classic\">Luc Didry</a>."
 
-#: lib/Lufi/Controller/Files.pm:236
+#: lib/Lufi/Controller/Files.pm:231
 msgid "The server was unable to find the file record to add your file part to. Please, contact the administrator."
 msgstr "The server was unable to find the file record to add your file part to. Please, contact the administrator."
 
-#: lib/Lufi/Controller/Files.pm:295
+#: lib/Lufi/Controller/Files.pm:290
 msgid "This file has been deactivated by the admins. Contact them to know why."
 msgstr "This file has been deactivated by the admins. Contact them to know why."
 
@@ -674,17 +674,17 @@ msgid "Unable to copy the link(s) to your clipboard"
 msgstr "Unable to copy the link(s) to your clipboard"
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:439
+#: lib/Lufi/Controller/Files.pm:434
 msgid "Unable to get counter for %1. The file does not exists. It will be removed from your localStorage."
 msgstr "Unable to get counter for %1. The file does not exists. It will be removed from your localStorage."
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:429
+#: lib/Lufi/Controller/Files.pm:424
 msgid "Unable to get counter for %1. The token is invalid."
 msgstr "Unable to get counter for %1. The token is invalid."
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:449
+#: lib/Lufi/Controller/Files.pm:444
 msgid "Unable to get counter for %1. You are not authenticated."
 msgstr "Unable to get counter for %1. You are not authenticated."
 
@@ -769,7 +769,7 @@ msgstr "Your browser does not have enough entropy to generate a strong encryptio
 msgid "Your file is too big: %1 (maximum size allowed: %2)"
 msgstr "Your file is too big: %1 (maximum size allowed: %2)"
 
-#: lib/Lufi/Controller/Files.pm:351
+#: lib/Lufi/Controller/Files.pm:346
 msgid "Your password is not valid. Please refresh the page to retry."
 msgstr "Your password is not valid. Please refresh the page to retry."
 

themes/default/lib/Lufi/I18N/lufi.pot

@@ -138,15 +138,15 @@ msgstr ""
 msgid "Copy to clipboard"
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:507
+#: lib/Lufi/Controller/Files.pm:502
 msgid "Could not delete the file. You are not authenticated."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:489
+#: lib/Lufi/Controller/Files.pm:484
 msgid "Could not find the file. Are you sure of the URL and the token?"
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:400
+#: lib/Lufi/Controller/Files.pm:395
 msgid "Could not find the file. Are you sure of the URL?"
 msgstr ""
 
@@ -222,15 +222,15 @@ msgstr ""
 msgid "Encrypting part XX1 of XX2"
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:289
+#: lib/Lufi/Controller/Files.pm:284
 msgid "Error: the file existed but was deleted."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:369
+#: lib/Lufi/Controller/Files.pm:364
 msgid "Error: the file has not been sent entirely."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:379
+#: lib/Lufi/Controller/Files.pm:374
 msgid "Error: unable to find the file. Are you sure of your URL?"
 msgstr ""
 
@@ -250,7 +250,7 @@ msgstr ""
 msgid "Export localStorage data"
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:471
+#: lib/Lufi/Controller/Files.pm:466
 msgid "File deleted"
 msgstr ""
 
@@ -436,7 +436,7 @@ msgid "Name of the zip file"
 msgstr ""
 
 #. (format_bytes($json->{size})
-#: lib/Lufi/Controller/Files.pm:108
+#: lib/Lufi/Controller/Files.pm:109
 msgid "No enough space available on the server for this file (size: %1)."
 msgstr ""
 
@@ -551,7 +551,7 @@ msgid "Sorry, your invitation has expired or has been deleted."
 msgstr ""
 
 #. ($invit->ldap_user_mail)
-#: lib/Lufi/Controller/Files.pm:122
+#: lib/Lufi/Controller/Files.pm:123
 msgid "Sorry, your invitation has expired or has been deleted. Please contact %1 to have another invitation."
 msgstr ""
 
@@ -585,7 +585,7 @@ msgstr ""
 msgid "The expiration delay (%1) is not between 1 and %2 days."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:468
+#: lib/Lufi/Controller/Files.pm:463
 msgid "The file has already been deleted"
 msgstr ""
 
@@ -640,11 +640,11 @@ msgstr ""
 msgid "The original (and only for now) author is <a href=\"https://fiat-tux.fr\" class=\"classic\">Luc Didry</a>."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:236
+#: lib/Lufi/Controller/Files.pm:231
 msgid "The server was unable to find the file record to add your file part to. Please, contact the administrator."
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:295
+#: lib/Lufi/Controller/Files.pm:290
 msgid "This file has been deactivated by the admins. Contact them to know why."
 msgstr ""
 
@@ -674,17 +674,17 @@ msgid "Unable to copy the link(s) to your clipboard"
 msgstr ""
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:439
+#: lib/Lufi/Controller/Files.pm:434
 msgid "Unable to get counter for %1. The file does not exists. It will be removed from your localStorage."
 msgstr ""
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:429
+#: lib/Lufi/Controller/Files.pm:424
 msgid "Unable to get counter for %1. The token is invalid."
 msgstr ""
 
 #. ($short)
-#: lib/Lufi/Controller/Files.pm:449
+#: lib/Lufi/Controller/Files.pm:444
 msgid "Unable to get counter for %1. You are not authenticated."
 msgstr ""
 
@@ -769,7 +769,7 @@ msgstr ""
 msgid "Your file is too big: %1 (maximum size allowed: %2)"
 msgstr ""
 
-#: lib/Lufi/Controller/Files.pm:351
+#: lib/Lufi/Controller/Files.pm:346
 msgid "Your password is not valid. Please refresh the page to retry."
 msgstr ""
 

utilities/migrations/mysql.sql

@@ -51,3 +51,7 @@ DROP TABLE invitations;
 ALTER TABLE files MODIFY filesize bigint;
 -- 4 down
 ALTER TABLE files MODIFY filesize integer;
+-- 5 up
+ALTER TABLE slices MODIFY path varchar(255);
+-- 5 down
+ALTER TABLE slices MODIFY path varchar(255) unique NOT NULL;

utilities/migrations/pg.sql

@@ -54,3 +54,9 @@ DROP TABLE invitations;
 ALTER TABLE files ALTER COLUMN filesize TYPE bigint;
 -- 5 down
 ALTER TABLE files ALTER COLUMN filesize TYPE integer;
+-- 6 up
+ALTER TABLE slices ALTER COLUMN path DROP NOT NULL;
+ALTER TABLE slices DROP CONSTRAINT slices_path_key;
+-- 6 down
+ALTER TABLE slices ADD CONSTRAINT slices_path_key UNIQUE (path) ;
+ALTER TABLE slices ALTER COLUMN path SET NOT NULL;