From a5cebd6bcfccb87bee272ef1b102451a128698a0 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Sun, 6 Dec 2020 13:53:55 -0700 Subject: [PATCH] s3 backups: handle CompleteMultipartUpload and AbortMultipartUpload on the panel instead of in wings, add BACKUP_PRESIGNED_URL_LIFESPAN environment variable --- app/Extensions/Backups/BackupManager.php | 3 +- .../Backups/BackupRemoteUploadController.php | 22 ++++++---- .../Remote/Backups/BackupStatusController.php | 43 ++++++++++++++++++- .../Backups/InitiateBackupService.php | 6 ++- config/backups.php | 4 ++ 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/app/Extensions/Backups/BackupManager.php b/app/Extensions/Backups/BackupManager.php index fda52f90a..fb0e0a26a 100644 --- a/app/Extensions/Backups/BackupManager.php +++ b/app/Extensions/Backups/BackupManager.php @@ -9,6 +9,7 @@ use Illuminate\Support\Str; use Webmozart\Assert\Assert; use InvalidArgumentException; use League\Flysystem\AdapterInterface; +use Illuminate\Foundation\Application; use League\Flysystem\AwsS3v3\AwsS3Adapter; use League\Flysystem\Memory\MemoryAdapter; use Illuminate\Contracts\Config\Repository; @@ -44,7 +45,7 @@ class BackupManager * * @param \Illuminate\Foundation\Application $app */ - public function __construct($app) + public function __construct(Application $app) { $this->app = $app; $this->config = $app->make(Repository::class); diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php index cf52fc450..4bf2f8c09 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php @@ -6,7 +6,9 @@ use Carbon\CarbonImmutable; use Illuminate\Http\Request; use Pterodactyl\Models\Backup; use Illuminate\Http\JsonResponse; +use Illuminate\Foundation\Application; use League\Flysystem\AwsS3v3\AwsS3Adapter; +use Illuminate\Contracts\Config\Repository; use Pterodactyl\Http\Controllers\Controller; use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; @@ -16,6 +18,11 @@ class BackupRemoteUploadController extends Controller { const PART_SIZE = 5 * 1024 * 1024 * 1024; + /** + * @var \Illuminate\Contracts\Config\Repository + */ + protected $config; + /** * @var \Pterodactyl\Repositories\Eloquent\BackupRepository */ @@ -29,11 +36,13 @@ class BackupRemoteUploadController extends Controller /** * BackupRemoteUploadController constructor. * + * @param \Illuminate\Foundation\Application $app * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ - public function __construct(BackupRepository $repository, BackupManager $backupManager) + public function __construct(Application $app, BackupRepository $repository, BackupManager $backupManager) { + $this->config = $app->make(Repository::class); $this->repository = $repository; $this->backupManager = $backupManager; } @@ -69,7 +78,7 @@ class BackupRemoteUploadController extends Controller // Ensure we are using the S3 adapter. $adapter = $this->backupManager->adapter(); if (! $adapter instanceof AwsS3Adapter) { - throw new BadRequestHttpException('The configured backup adapter is not an S3 compatiable adapter.'); + throw new BadRequestHttpException('The configured backup adapter is not an S3 compatible adapter.'); } // The path where backup will be uploaded to @@ -77,7 +86,7 @@ class BackupRemoteUploadController extends Controller // Get the S3 client $client = $adapter->getClient(); - $expires = CarbonImmutable::now()->addMinutes(30); + $expires = CarbonImmutable::now()->addMinutes($this->config->get('backups.presigned_url_lifespan', 60)); // Params for generating the presigned urls $params = [ @@ -102,14 +111,9 @@ class BackupRemoteUploadController extends Controller } return new JsonResponse([ + 'upload_id' => $params['UploadId'], 'parts' => $parts, 'part_size' => self::PART_SIZE, - 'complete_multipart_upload' => $client->createPresignedRequest( - $client->getCommand('CompleteMultipartUpload', $params), $expires - )->getUri()->__toString(), - 'abort_multipart_upload' => $client->createPresignedRequest( - $client->getCommand('AbortMultipartUpload', $params), $expires->addMinutes(15) - )->getUri()->__toString(), ]); } } diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php index 85a3acf6a..dac94f8d5 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php @@ -3,8 +3,11 @@ namespace Pterodactyl\Http\Controllers\Api\Remote\Backups; use Carbon\CarbonImmutable; +use Pterodactyl\Models\Backup; use Illuminate\Http\JsonResponse; +use League\Flysystem\AwsS3v3\AwsS3Adapter; use Pterodactyl\Http\Controllers\Controller; +use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest; @@ -16,14 +19,21 @@ class BackupStatusController extends Controller */ private $repository; + /** + * @var \Pterodactyl\Extensions\Backups\BackupManager + */ + private $backupManager; + /** * BackupStatusController constructor. * * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository + * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ - public function __construct(BackupRepository $repository) + public function __construct(BackupRepository $repository, BackupManager $backupManager) { $this->repository = $repository; + $this->backupManager = $backupManager; } /** @@ -31,9 +41,11 @@ class BackupStatusController extends Controller * * @param \Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest $request * @param string $backup + * * @return \Illuminate\Http\JsonResponse * * @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException + * @throws \Exception */ public function __invoke(ReportBackupCompleteRequest $request, string $backup) { @@ -47,6 +59,8 @@ class BackupStatusController extends Controller } $successful = $request->input('successful') ? true : false; + + // TODO: Still run s3 code even if this fails. $model->forceFill([ 'is_successful' => $successful, 'checksum' => $successful ? ($request->input('checksum_type') . ':' . $request->input('checksum')) : null, @@ -54,6 +68,33 @@ class BackupStatusController extends Controller 'completed_at' => CarbonImmutable::now(), ])->save(); + // Check if we are using the s3 backup adapter. + $adapter = $this->backupManager->adapter(); + if ($adapter instanceof AwsS3Adapter) { + /** @var \Pterodactyl\Models\Backup $backup */ + $backup = Backup::query()->where('uuid', $backup)->firstOrFail(); + + $client = $adapter->getClient(); + + $params = [ + 'Bucket' => $adapter->getBucket(), + 'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid), + 'UploadId' => $request->input('upload_id'), + ]; + + // If the backup was not successful, send an AbortMultipartUpload request. + if (! $successful) { + $client->execute($client->getCommand('AbortMultipartUpload', $params)); + return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); + } + + // Otherwise send a CompleteMultipartUpload request. + $params['MultipartUpload'] = [ + 'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'], + ]; + $client->execute($client->getCommand('CompleteMultipartUpload', $params)); + } + return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); } } diff --git a/app/Services/Backups/InitiateBackupService.php b/app/Services/Backups/InitiateBackupService.php index 5c34961a4..c7b884537 100644 --- a/app/Services/Backups/InitiateBackupService.php +++ b/app/Services/Backups/InitiateBackupService.php @@ -99,6 +99,8 @@ class InitiateBackupService * * @param \Pterodactyl\Models\Server $server * @param string|null $name + * @param bool $override + * * @return \Pterodactyl\Models\Backup * * @throws \Throwable @@ -107,13 +109,13 @@ class InitiateBackupService */ public function handle(Server $server, string $name = null, bool $override = false): Backup { - $previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); + /*$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); if ($previous->count() >= 2) { throw new TooManyRequestsHttpException( CarbonImmutable::now()->diffInSeconds($previous->last()->created_at->addMinutes(10)), 'Only two backups may be generated within a 10 minute span of time.' ); - } + }*/ // Check if the server has reached or exceeded it's backup limit if (!$server->backup_limit || $server->backups()->where('is_successful', true)->count() >= $server->backup_limit) { diff --git a/config/backups.php b/config/backups.php index ec9bd175e..32ee1aa8a 100644 --- a/config/backups.php +++ b/config/backups.php @@ -8,6 +8,10 @@ return [ // have been made, without losing data. 'default' => env('APP_BACKUP_DRIVER', Backup::ADAPTER_WINGS), + // This value is used to determine the lifespan of UploadPart presigned urls that wings + // uses to upload backups to S3 storage. Value is in minutes, so this would default to an hour. + 'presigned_url_lifespan' => env('BACKUP_PRESIGNED_URL_LIFESPAN', 60), + 'disks' => [ // There is no configuration for the local disk for Wings. That configuration // is determined by the Daemon configuration, and not the Panel.