s3 backups: handle CompleteMultipartUpload and AbortMultipartUpload on the panel instead of in wings, add BACKUP_PRESIGNED_URL_LIFESPAN environment variable
This commit is contained in:
Matthew Penner
parent
5d23d894ae
commit
a5cebd6bcf
5 changed files with 65 additions and 13 deletions
|
|
@ -9,6 +9,7 @@ use Illuminate\Support\Str;
|
|||
use Webmozart\Assert\Assert;
|
||||
use InvalidArgumentException;
|
||||
use League\Flysystem\AdapterInterface;
|
||||
use Illuminate\Foundation\Application;
|
||||
use League\Flysystem\AwsS3v3\AwsS3Adapter;
|
||||
use League\Flysystem\Memory\MemoryAdapter;
|
||||
use Illuminate\Contracts\Config\Repository;
|
||||
|
|
@ -44,7 +45,7 @@ class BackupManager
|
|||
*
|
||||
* @param \Illuminate\Foundation\Application $app
|
||||
*/
|
||||
public function __construct($app)
|
||||
public function __construct(Application $app)
|
||||
{
|
||||
$this->app = $app;
|
||||
$this->config = $app->make(Repository::class);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ use Carbon\CarbonImmutable;
|
|||
use Illuminate\Http\Request;
|
||||
use Pterodactyl\Models\Backup;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use Illuminate\Foundation\Application;
|
||||
use League\Flysystem\AwsS3v3\AwsS3Adapter;
|
||||
use Illuminate\Contracts\Config\Repository;
|
||||
use Pterodactyl\Http\Controllers\Controller;
|
||||
use Pterodactyl\Extensions\Backups\BackupManager;
|
||||
use Pterodactyl\Repositories\Eloquent\BackupRepository;
|
||||
|
|
@ -16,6 +18,11 @@ class BackupRemoteUploadController extends Controller
|
|||
{
|
||||
const PART_SIZE = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* @var \Illuminate\Contracts\Config\Repository
|
||||
*/
|
||||
protected $config;
|
||||
|
||||
/**
|
||||
* @var \Pterodactyl\Repositories\Eloquent\BackupRepository
|
||||
*/
|
||||
|
|
@ -29,11 +36,13 @@ class BackupRemoteUploadController extends Controller
|
|||
/**
|
||||
* BackupRemoteUploadController constructor.
|
||||
*
|
||||
* @param \Illuminate\Foundation\Application $app
|
||||
* @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
|
||||
* @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
|
||||
*/
|
||||
public function __construct(BackupRepository $repository, BackupManager $backupManager)
|
||||
public function __construct(Application $app, BackupRepository $repository, BackupManager $backupManager)
|
||||
{
|
||||
$this->config = $app->make(Repository::class);
|
||||
$this->repository = $repository;
|
||||
$this->backupManager = $backupManager;
|
||||
}
|
||||
|
|
@ -69,7 +78,7 @@ class BackupRemoteUploadController extends Controller
|
|||
// Ensure we are using the S3 adapter.
|
||||
$adapter = $this->backupManager->adapter();
|
||||
if (! $adapter instanceof AwsS3Adapter) {
|
||||
throw new BadRequestHttpException('The configured backup adapter is not an S3 compatiable adapter.');
|
||||
throw new BadRequestHttpException('The configured backup adapter is not an S3 compatible adapter.');
|
||||
}
|
||||
|
||||
// The path where backup will be uploaded to
|
||||
|
|
@ -77,7 +86,7 @@ class BackupRemoteUploadController extends Controller
|
|||
|
||||
// Get the S3 client
|
||||
$client = $adapter->getClient();
|
||||
$expires = CarbonImmutable::now()->addMinutes(30);
|
||||
$expires = CarbonImmutable::now()->addMinutes($this->config->get('backups.presigned_url_lifespan', 60));
|
||||
|
||||
// Params for generating the presigned urls
|
||||
$params = [
|
||||
|
|
@ -102,14 +111,9 @@ class BackupRemoteUploadController extends Controller
|
|||
}
|
||||
|
||||
return new JsonResponse([
|
||||
'upload_id' => $params['UploadId'],
|
||||
'parts' => $parts,
|
||||
'part_size' => self::PART_SIZE,
|
||||
'complete_multipart_upload' => $client->createPresignedRequest(
|
||||
$client->getCommand('CompleteMultipartUpload', $params), $expires
|
||||
)->getUri()->__toString(),
|
||||
'abort_multipart_upload' => $client->createPresignedRequest(
|
||||
$client->getCommand('AbortMultipartUpload', $params), $expires->addMinutes(15)
|
||||
)->getUri()->__toString(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,11 @@
|
|||
namespace Pterodactyl\Http\Controllers\Api\Remote\Backups;
|
||||
|
||||
use Carbon\CarbonImmutable;
|
||||
use Pterodactyl\Models\Backup;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use League\Flysystem\AwsS3v3\AwsS3Adapter;
|
||||
use Pterodactyl\Http\Controllers\Controller;
|
||||
use Pterodactyl\Extensions\Backups\BackupManager;
|
||||
use Pterodactyl\Repositories\Eloquent\BackupRepository;
|
||||
use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
|
||||
use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest;
|
||||
|
|
@ -16,14 +19,21 @@ class BackupStatusController extends Controller
|
|||
*/
|
||||
private $repository;
|
||||
|
||||
/**
|
||||
* @var \Pterodactyl\Extensions\Backups\BackupManager
|
||||
*/
|
||||
private $backupManager;
|
||||
|
||||
/**
|
||||
* BackupStatusController constructor.
|
||||
*
|
||||
* @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository
|
||||
* @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager
|
||||
*/
|
||||
public function __construct(BackupRepository $repository)
|
||||
public function __construct(BackupRepository $repository, BackupManager $backupManager)
|
||||
{
|
||||
$this->repository = $repository;
|
||||
$this->backupManager = $backupManager;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -31,9 +41,11 @@ class BackupStatusController extends Controller
|
|||
*
|
||||
* @param \Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest $request
|
||||
* @param string $backup
|
||||
*
|
||||
* @return \Illuminate\Http\JsonResponse
|
||||
*
|
||||
* @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function __invoke(ReportBackupCompleteRequest $request, string $backup)
|
||||
{
|
||||
|
|
@ -47,6 +59,8 @@ class BackupStatusController extends Controller
|
|||
}
|
||||
|
||||
$successful = $request->input('successful') ? true : false;
|
||||
|
||||
// TODO: Still run s3 code even if this fails.
|
||||
$model->forceFill([
|
||||
'is_successful' => $successful,
|
||||
'checksum' => $successful ? ($request->input('checksum_type') . ':' . $request->input('checksum')) : null,
|
||||
|
|
@ -54,6 +68,33 @@ class BackupStatusController extends Controller
|
|||
'completed_at' => CarbonImmutable::now(),
|
||||
])->save();
|
||||
|
||||
// Check if we are using the s3 backup adapter.
|
||||
$adapter = $this->backupManager->adapter();
|
||||
if ($adapter instanceof AwsS3Adapter) {
|
||||
/** @var \Pterodactyl\Models\Backup $backup */
|
||||
$backup = Backup::query()->where('uuid', $backup)->firstOrFail();
|
||||
|
||||
$client = $adapter->getClient();
|
||||
|
||||
$params = [
|
||||
'Bucket' => $adapter->getBucket(),
|
||||
'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid),
|
||||
'UploadId' => $request->input('upload_id'),
|
||||
];
|
||||
|
||||
// If the backup was not successful, send an AbortMultipartUpload request.
|
||||
if (! $successful) {
|
||||
$client->execute($client->getCommand('AbortMultipartUpload', $params));
|
||||
return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);
|
||||
}
|
||||
|
||||
// Otherwise send a CompleteMultipartUpload request.
|
||||
$params['MultipartUpload'] = [
|
||||
'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'],
|
||||
];
|
||||
$client->execute($client->getCommand('CompleteMultipartUpload', $params));
|
||||
}
|
||||
|
||||
return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,6 +99,8 @@ class InitiateBackupService
|
|||
*
|
||||
* @param \Pterodactyl\Models\Server $server
|
||||
* @param string|null $name
|
||||
* @param bool $override
|
||||
*
|
||||
* @return \Pterodactyl\Models\Backup
|
||||
*
|
||||
* @throws \Throwable
|
||||
|
|
@ -107,13 +109,13 @@ class InitiateBackupService
|
|||
*/
|
||||
public function handle(Server $server, string $name = null, bool $override = false): Backup
|
||||
{
|
||||
$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10);
|
||||
/*$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10);
|
||||
if ($previous->count() >= 2) {
|
||||
throw new TooManyRequestsHttpException(
|
||||
CarbonImmutable::now()->diffInSeconds($previous->last()->created_at->addMinutes(10)),
|
||||
'Only two backups may be generated within a 10 minute span of time.'
|
||||
);
|
||||
}
|
||||
}*/
|
||||
|
||||
// Check if the server has reached or exceeded it's backup limit
|
||||
if (!$server->backup_limit || $server->backups()->where('is_successful', true)->count() >= $server->backup_limit) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,10 @@ return [
|
|||
// have been made, without losing data.
|
||||
'default' => env('APP_BACKUP_DRIVER', Backup::ADAPTER_WINGS),
|
||||
|
||||
// This value is used to determine the lifespan of UploadPart presigned urls that wings
|
||||
// uses to upload backups to S3 storage. Value is in minutes, so this would default to an hour.
|
||||
'presigned_url_lifespan' => env('BACKUP_PRESIGNED_URL_LIFESPAN', 60),
|
||||
|
||||
'disks' => [
|
||||
// There is no configuration for the local disk for Wings. That configuration
|
||||
// is determined by the Daemon configuration, and not the Panel.
|
||||
|
|
|
|||
Loading…
Reference in a new issue