kio-s3/src/s3backend.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2020 Elvis Angelaccio <elvis.angelaccio@kde.org>
0003     SPDX-License-Identifier: GPL-2.0-or-later
0004 */
0005
0006 #include "s3backend.h"
0007 #include "s3debug.h"
0008 #include "kio_s3.h"
0009
0010 #include <KLocalizedString>
0011
0012 #include <aws/core/auth/AWSCredentialsProvider.h>
0013 #include <aws/core/Aws.h>
0014 #include <aws/s3/model/Bucket.h>
0015 #include <aws/s3/model/CopyObjectRequest.h>
0016 #include <aws/s3/model/DeleteObjectRequest.h>
0017 #include <aws/s3/model/GetObjectRequest.h>
0018 #include <aws/s3/model/HeadObjectRequest.h>
0019 #include <aws/s3/model/ListObjectsV2Request.h>
0020 #include <aws/s3/model/PutObjectRequest.h>
0021
0022 #include <array>
0023
0024 static KIO::WorkerResult invalidUrlError() {
0025     static const auto s_invalidUrlError = KIO::WorkerResult::fail(
0026         KIO::ERR_WORKER_DEFINED,
0027         xi18nc("@info", "Invalid S3 URI, bucket name is missing from the host.<nl/>A valid S3 URI must be written in the form: <link>s3://bucket/key</link>")
0028     );
0029
0030     return s_invalidUrlError;
0031 }
0032
0033 S3Backend::S3Backend(S3Worker *q)
0034     : q(q)
0035 {
0036     Aws::SDKOptions options;
0037     Aws::InitAPI(options);
0038
0039     m_configProfileName = Aws::Auth::GetConfigProfileFilename();
0040     qCDebug(S3) << "S3 backend initialized, config profile name:" << m_configProfileName.c_str();
0041 }
0042
0043 KIO::WorkerResult S3Backend::listDir(const QUrl &url)
0044 {
0045     const auto s3url = S3Url(url);
0046     qCDebug(S3) << "Going to list" << s3url;
0047
0048     if (s3url.isRoot()) {
0049         const bool hasBuckets = listBuckets();
0050         listCwdEntry(ReadOnlyCwd);
0051         if (hasBuckets) {
0052             return KIO::WorkerResult::pass();
0053         }
0054         return KIO::WorkerResult::fail(KIO::ERR_WORKER_DEFINED, xi18nc("@info", "Could not find S3 buckets, please check your AWS configuration."));
0055     }
0056
0057     if (s3url.isBucket()) {
0058         listBucket(s3url.BucketName());
0059         listCwdEntry();
0060         return KIO::WorkerResult::pass();
0061     }
0062
0063     if (!s3url.isKey()) {
0064         qCDebug(S3) << "Could not list invalid S3 url:" << url;
0065         return invalidUrlError();
0066     }
0067
0068     Q_ASSERT(s3url.isKey());
0069
0070     listKey(s3url);
0071     listCwdEntry();
0072     return KIO::WorkerResult::pass();
0073 }
0074
0075 KIO::WorkerResult S3Backend::stat(const QUrl &url)
0076 {
0077     const auto s3url = S3Url(url);
0078     qCDebug(S3) << "Going to stat()" << s3url;
0079
0080     if (s3url.isRoot()) {
0081         return KIO::WorkerResult::pass();
0082     }
0083
0084     if (s3url.isBucket()) {
0085         KIO::UDSEntry entry;
0086         entry.reserve(4);
0087         entry.fastInsert(KIO::UDSEntry::UDS_NAME, s3url.bucketName());
0088         entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0089         entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR | S_IXGRP | S_IXOTH);
0090         entry.fastInsert(KIO::UDSEntry::UDS_MIME_TYPE, QStringLiteral("inode/directory"));
0091         q->statEntry(entry);
0092         return KIO::WorkerResult::pass();
0093     }
0094
0095     if (!s3url.isKey()) {
0096         qCDebug(S3) << "Could not stat invalid S3 url:" << url;
0097         return invalidUrlError();
0098     }
0099
0100     Q_ASSERT(s3url.isKey());
0101
0102     // Try to do an HEAD request for the key.
0103     // If the URL is a folder, S3 will reply only if there is a 0-sized object with that key.
0104     const auto pathComponents = url.path().split(QLatin1Char('/'), Qt::SkipEmptyParts);
0105     // The URL could be s3://<bucketName>/ which would have "/" as path(). Fallback to bucketName in that case.
0106     const bool isRootKey = pathComponents.isEmpty();
0107     const auto fileName = isRootKey ? s3url.bucketName() : pathComponents.last();
0108
0109     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0110     const Aws::S3::S3Client client(clientConfiguration);
0111
0112     Aws::S3::Model::HeadObjectRequest headObjectRequest;
0113     headObjectRequest.SetBucket(s3url.BucketName());
0114     headObjectRequest.SetKey(s3url.Key());
0115
0116     auto headObjectRequestOutcome = client.HeadObject(headObjectRequest);
0117     if (!isRootKey && headObjectRequestOutcome.IsSuccess()) {
0118         Aws::String contentType = headObjectRequestOutcome.GetResult().GetContentType();
0119         // This is set by S3 when creating a 0-sized folder from the AWS console. Use the freedesktop mimetype instead.
0120         if (contentType == "application/x-directory") {
0121             contentType = "inode/directory";
0122         }
0123         const bool isDir = contentType == "inode/directory";
0124         KIO::UDSEntry entry;
0125         entry.reserve(7);
0126         entry.fastInsert(KIO::UDSEntry::UDS_NAME, fileName);
0127         entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, fileName);
0128         entry.fastInsert(KIO::UDSEntry::UDS_MIME_TYPE, QString::fromLatin1(contentType.c_str(), contentType.size()));
0129         entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, isDir ? S_IFDIR : S_IFREG);
0130         entry.fastInsert(KIO::UDSEntry::UDS_SIZE, headObjectRequestOutcome.GetResult().GetContentLength());
0131         if (isDir) {
0132             entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0133         } else {
0134             // For keys we would need another request (GetObjectAclRequest) to get the permission,
0135             // but it is kind of pointless to map the AWS ACL model to UNIX permission anyway.
0136             // So assume keys are always writable, we'll handle the failure if they are not.
0137             // The same logic will be applied to all the other UDS_ACCESS instances for keys.
0138             entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
0139             const auto lastModifiedTime = headObjectRequestOutcome.GetResult().GetLastModified();
0140             entry.fastInsert(KIO::UDSEntry::UDS_MODIFICATION_TIME, lastModifiedTime.SecondsWithMSPrecision());
0141         }
0142
0143         q->statEntry(entry);
0144     } else {
0145         if (!isRootKey) {
0146             qCDebug(S3).nospace() << "Could not get HEAD object for key: " << s3url.key() << " - " << headObjectRequestOutcome.GetError().GetMessage().c_str() << " - assuming it's a folder.";
0147         }
0148         // HACK: assume this is a folder (i.e. a virtual key without associated object).
0149         // If it were a key or a 0-sized folder the HEAD request would likely have worked.
0150         // This is needed to upload local folders to S3.
0151         KIO::UDSEntry entry;
0152         entry.reserve(6);
0153         entry.fastInsert(KIO::UDSEntry::UDS_NAME, fileName);
0154         entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, fileName);
0155         entry.fastInsert(KIO::UDSEntry::UDS_MIME_TYPE, QStringLiteral("inode/directory"));
0156         entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0157         entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0158         entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0159         q->statEntry(entry);
0160     }
0161
0162     return KIO::WorkerResult::pass();
0163 }
0164
0165 KIO::WorkerResult S3Backend::mimetype(const QUrl &url)
0166 {
0167     const auto s3url = S3Url(url);
0168     qCDebug(S3) << "Going to get mimetype for" << s3url;
0169
0170     q->mimeType(contentType(s3url));
0171     return KIO::WorkerResult::pass();
0172 }
0173
0174 KIO::WorkerResult S3Backend::get(const QUrl &url)
0175 {
0176     const auto s3url = S3Url(url);
0177     qCDebug(S3) << "Going to get" << s3url;
0178
0179     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0180     const Aws::S3::S3Client client(clientConfiguration);
0181
0182     q->mimeType(contentType(s3url));
0183
0184     Aws::S3::Model::GetObjectRequest objectRequest;
0185     objectRequest.SetBucket(s3url.BucketName());
0186     objectRequest.SetKey(s3url.Key());
0187
0188     auto getObjectOutcome = client.GetObject(objectRequest);
0189     if (getObjectOutcome.IsSuccess()) {
0190         auto objectResult = getObjectOutcome.GetResultWithOwnership();
0191         auto& retrievedFile = objectResult.GetBody();
0192         qCDebug(S3) << "Key" << s3url.key() << "has Content-Length:" << objectResult.GetContentLength();
0193         q->totalSize(objectResult.GetContentLength());
0194
0195         std::array<char, 1024*1024> buffer{};
0196         while (!retrievedFile.eof()) {
0197             const auto readBytes = retrievedFile.read(buffer.data(), buffer.size()).gcount();
0198             if (readBytes > 0) {
0199                 q->data(QByteArray(buffer.data(), readBytes));
0200             }
0201         }
0202
0203         q->data(QByteArray());
0204
0205     } else {
0206         // NOTE: normally we shouldn't get this error, because KIO does a stat() before the get() and if the url doesn't exist, our stat() assumes it's a folder.
0207         // This is why this error is not shown to users in the frontend.
0208         qCWarning(S3) << "Could not get object with key:" << s3url.key() << " - " << getObjectOutcome.GetError().GetMessage().c_str();
0209     }
0210
0211     return KIO::WorkerResult::pass();
0212 }
0213
0214 KIO::WorkerResult S3Backend::put(const QUrl &url, int permissions, KIO::JobFlags flags)
0215 {
0216     Q_UNUSED(permissions)
0217     Q_UNUSED(flags)
0218     const auto s3url = S3Url(url);
0219     qCDebug(S3) << "Going to upload data to" << s3url;
0220
0221     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0222     const Aws::S3::S3Client client(clientConfiguration);
0223
0224     Aws::S3::Model::PutObjectRequest request;
0225     request.SetBucket(s3url.BucketName());
0226     request.SetKey(s3url.Key());
0227
0228     const auto putDataStream = std::make_shared<Aws::StringStream>("");
0229
0230     int bytesCount = 0;
0231     int n;
0232     do {
0233         QByteArray buffer;
0234         q->dataReq();
0235         n = q->readData(buffer);
0236         if (!buffer.isEmpty()) {
0237             bytesCount += n;
0238             putDataStream->write(buffer.data(), n);
0239         }
0240     } while (n > 0);
0241
0242     if (n < 0) {
0243         qCWarning(S3) << "Failed to upload data to" << s3url;
0244         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_WRITE, url.toDisplayString());
0245     }
0246
0247     request.SetBody(putDataStream);
0248
0249     auto putObjectOutcome = client.PutObject(request);
0250     if (putObjectOutcome.IsSuccess()) {
0251         qCDebug(S3) << "Uploaded" <<  bytesCount << "bytes to key:" << s3url.key();
0252     } else {
0253         qCDebug(S3) << "Could not PUT object with key:" << s3url.key() << " - " << putObjectOutcome.GetError().GetMessage().c_str();
0254     }
0255
0256     return KIO::WorkerResult::pass();
0257 }
0258
0259 KIO::WorkerResult S3Backend::copy(const QUrl &src, const QUrl &dest, int permissions, KIO::JobFlags flags)
0260 {
0261     Q_UNUSED(permissions)
0262     Q_UNUSED(flags)
0263
0264     const auto s3src = S3Url(src);
0265     const auto s3dest = S3Url(dest);
0266     qCDebug(S3) << "Going to copy" << s3src << "to" << s3dest;
0267
0268     if (src == dest) {
0269         return KIO::WorkerResult::fail(KIO::ERR_FILE_ALREADY_EXIST, QString());
0270     }
0271
0272     if (s3src.isRoot() || s3src.isBucket()) {
0273         qCDebug(S3) << "Cannot copy from root or bucket url:" << src;
0274         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_OPEN_FOR_READING, src.toDisplayString());
0275     }
0276
0277     if (!s3src.isKey()) {
0278         qCDebug(S3) << "Cannot copy from invalid S3 url:" << src;
0279         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_OPEN_FOR_READING, src.toDisplayString());
0280     }
0281
0282     // TODO: can we copy to isBucket() urls?
0283     if (s3dest.isRoot() || s3dest.isBucket()) {
0284         qCDebug(S3) << "Cannot copy to root or bucket url:" << dest;
0285         return KIO::WorkerResult::fail(KIO::ERR_WRITE_ACCESS_DENIED, dest.toDisplayString());
0286     }
0287
0288     if (!s3dest.isKey()) {
0289         qCDebug(S3) << "Cannot write to invalid S3 url:" << dest;
0290         return KIO::WorkerResult::fail(KIO::ERR_WRITE_ACCESS_DENIED, dest.toDisplayString());
0291     }
0292
0293     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0294     const Aws::S3::S3Client client(clientConfiguration);
0295
0296     // Check if destination key already exists, otherwise S3 will overwrite it leading to data loss.
0297     Aws::S3::Model::HeadObjectRequest headObjectRequest;
0298     headObjectRequest.SetBucket(s3dest.BucketName());
0299     headObjectRequest.SetKey(s3dest.Key());
0300     auto headObjectRequestOutcome = client.HeadObject(headObjectRequest);
0301     if (headObjectRequestOutcome.IsSuccess()) {
0302         return KIO::WorkerResult::fail(KIO::ERR_FILE_ALREADY_EXIST, QString());
0303     }
0304
0305     Aws::S3::Model::CopyObjectRequest request;
0306     request.SetCopySource(s3src.BucketName() + "/" + s3src.Key());
0307     request.SetBucket(s3dest.BucketName());
0308     request.SetKey(s3dest.Key());
0309
0310     auto copyObjectOutcome = client.CopyObject(request);
0311     if (!copyObjectOutcome.IsSuccess()) {
0312         qCDebug(S3) << "Could not copy" << src << "to" << dest << "- " << copyObjectOutcome.GetError().GetMessage().c_str();
0313         return KIO::WorkerResult::fail(KIO::ERR_WORKER_DEFINED, xi18nc("@info", "Could not copy <link>%1</link> to <link>%2</link>", src.toDisplayString(), dest.toDisplayString()));
0314     }
0315
0316     return KIO::WorkerResult::pass();
0317 }
0318
0319 KIO::WorkerResult S3Backend::mkdir(const QUrl &url, int permissions)
0320 {
0321     Q_UNUSED(url)
0322     Q_UNUSED(permissions)
0323     qCDebug(S3) << "Pretending creation of folder" << url;
0324     return KIO::WorkerResult::pass();
0325 }
0326
0327 KIO::WorkerResult S3Backend::del(const QUrl &url, bool isFile)
0328 {
0329     Q_UNUSED(isFile)
0330     const auto s3url = S3Url(url);
0331     qCDebug(S3) << "Going to delete" << s3url;
0332
0333     if (s3url.isRoot() || s3url.isBucket()) {
0334         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_DELETE, url.toDisplayString());
0335     }
0336
0337     if (!s3url.isKey()) {
0338         return invalidUrlError();
0339     }
0340
0341     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0342     const Aws::S3::S3Client client(clientConfiguration);
0343
0344     // Start recursive delete by using the root prefix.
0345     if (deletePrefix(client, s3url, s3url.Prefix())) {
0346         return KIO::WorkerResult::pass();
0347     } else {
0348         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_DELETE, url.toDisplayString());
0349     }
0350 }
0351
0352 KIO::WorkerResult S3Backend::rename(const QUrl &src, const QUrl &dest, KIO::JobFlags flags)
0353 {
0354     Q_UNUSED(flags)
0355     qCDebug(S3) << "Going to rename" << src << "to" << dest;
0356
0357     // FIXME: rename of virtual folders doesn't work, because folders don't exist in S3.
0358     // This would require some special handling:
0359     // 1. detect that src is a folder
0360     // 2. list the folder
0361     // 3. rename each key listed
0362     // Workaround: copy+delete from dolphin...
0363
0364     const auto copyResult = copy(src, dest, -1, flags);
0365     if (!copyResult.success()) {
0366         qCDebug(S3).nospace() << "Could not copy " << src << " to " << dest << ", aborting rename()";
0367         if (copyResult.error() == KIO::ERR_FILE_ALREADY_EXIST) {
0368             return copyResult;
0369         }
0370         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_RENAME, src.toDisplayString());
0371     }
0372
0373     const auto delResult = del(src, false);
0374     if (!delResult.success()) {
0375         qCDebug(S3) << "Could not delete" << src << "after it was copied to" << dest;
0376         return KIO::WorkerResult::fail(KIO::ERR_CANNOT_RENAME, src.toDisplayString());
0377     }
0378
0379     return KIO::WorkerResult::pass();
0380 }
0381
0382 bool S3Backend::listBuckets()
0383 {
0384     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0385     const Aws::S3::S3Client client(clientConfiguration);
0386     const auto listBucketsOutcome = client.ListBuckets();
0387     bool hasBuckets = false;
0388
0389     if (listBucketsOutcome.IsSuccess()) {
0390         const auto buckets = listBucketsOutcome.GetResult().GetBuckets();
0391         hasBuckets = !buckets.empty();
0392         for (const auto &bucket : buckets) {
0393             const auto bucketName = QString::fromLatin1(bucket.GetName().c_str(), bucket.GetName().size());
0394             qCDebug(S3) << "Found bucket:" << bucketName;
0395             KIO::UDSEntry entry;
0396             entry.reserve(7);
0397             entry.fastInsert(KIO::UDSEntry::UDS_NAME, bucketName);
0398             entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, bucketName);
0399             entry.fastInsert(KIO::UDSEntry::UDS_URL, QStringLiteral("s3://%1/").arg(bucketName));
0400             entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0401             entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0402             entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0403             entry.fastInsert(KIO::UDSEntry::UDS_ICON_NAME, QStringLiteral("folder-network"));
0404             q->listEntry(entry);
0405         }
0406     } else {
0407         qCDebug(S3) << "Could not list buckets:" << listBucketsOutcome.GetError().GetMessage().c_str();
0408     }
0409
0410     return hasBuckets;
0411 }
0412
0413 void S3Backend::listBucket(const Aws::String &bucketName)
0414 {
0415     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0416     const Aws::S3::S3Client client(clientConfiguration);
0417
0418     Aws::S3::Model::ListObjectsV2Request listObjectsRequest;
0419     listObjectsRequest.SetBucket(bucketName);
0420     listObjectsRequest.SetDelimiter("/");
0421
0422     qCDebug(S3) << "Listing objects in bucket" << bucketName.c_str() << "...";
0423     const auto listObjectsOutcome = client.ListObjectsV2(listObjectsRequest);
0424     if (listObjectsOutcome.IsSuccess()) {
0425
0426         const auto bucket = QString::fromLatin1(bucketName.c_str(), bucketName.size());
0427         const auto objects = listObjectsOutcome.GetResult().GetContents();
0428         for (const auto &object : objects) {
0429             KIO::UDSEntry entry;
0430             const auto objectKey = QString::fromUtf8(object.GetKey().c_str());
0431             entry.reserve(6);
0432             entry.fastInsert(KIO::UDSEntry::UDS_NAME, objectKey);
0433             entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, objectKey);
0434             entry.fastInsert(KIO::UDSEntry::UDS_URL, QStringLiteral("s3://%1/%2").arg(bucket, objectKey));
0435             entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFREG);
0436             entry.fastInsert(KIO::UDSEntry::UDS_SIZE, object.GetSize());
0437             entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
0438             q->listEntry(entry);
0439         }
0440
0441         const auto commonPrefixes = listObjectsOutcome.GetResult().GetCommonPrefixes();
0442         for (const auto &commonPrefix : commonPrefixes) {
0443             KIO::UDSEntry entry;
0444             QString prefix = QString::fromUtf8(commonPrefix.GetPrefix().c_str(), commonPrefix.GetPrefix().size());
0445             if (prefix.endsWith(QLatin1Char('/'))) {
0446                 prefix.chop(1);
0447             }
0448             entry.reserve(6);
0449             entry.fastInsert(KIO::UDSEntry::UDS_NAME, prefix);
0450             entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, prefix);
0451             entry.fastInsert(KIO::UDSEntry::UDS_URL, QStringLiteral("s3://%1/%2/").arg(bucket, prefix));
0452             entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0453             entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0454             entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0455
0456             q->listEntry(entry);
0457         }
0458     } else {
0459         qCDebug(S3) << "Could not list bucket: " << listObjectsOutcome.GetError().GetMessage().c_str();
0460     }
0461 }
0462
0463 void S3Backend::listKey(const S3Url &s3url)
0464 {
0465     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0466     const Aws::S3::S3Client client(clientConfiguration);
0467
0468     const QString prefix = s3url.prefix();
0469
0470     Aws::S3::Model::ListObjectsV2Request listObjectsRequest;
0471     listObjectsRequest.SetBucket(s3url.BucketName());
0472     listObjectsRequest.SetDelimiter("/");
0473     listObjectsRequest.SetPrefix(s3url.Prefix());
0474
0475     qCDebug(S3) << "Listing prefix" << prefix << "...";
0476     const auto listObjectsOutcome = client.ListObjectsV2(listObjectsRequest);
0477     if (listObjectsOutcome.IsSuccess()) {
0478         const auto objects = listObjectsOutcome.GetResult().GetContents();
0479         // TODO: handle listObjectsOutcome.GetResult().GetIsTruncated()
0480         // By default the max-keys request parameter is 1000, which is reasonable for us
0481         // since we filter the keys by the name of the folder, but it won't work
0482         // if someone has very big folders with more than 1000 files.
0483         qCDebug(S3) << "Prefix" << prefix << "has" << objects.size() << "objects";
0484         for (const auto &object : objects) {
0485             QString key = QString::fromUtf8(object.GetKey().c_str(), object.GetKey().size());
0486             // Note: key might be empty. 0-sized virtual folders have object.GetKey() equal to prefix.
0487             key.remove(0, prefix.length());
0488
0489             KIO::UDSEntry entry;
0490             // S3 always appends trailing slash to "folder" objects.
0491             if (key.endsWith(QLatin1Char('/'))) {
0492                 entry.reserve(5);
0493                 entry.fastInsert(KIO::UDSEntry::UDS_NAME, key);
0494                 entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, key);
0495                 entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0496                 entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0497                 entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0498                 q->listEntry(entry);
0499             } else if (!key.isEmpty()) { // Not a folder.
0500                 entry.reserve(6);
0501                 entry.fastInsert(KIO::UDSEntry::UDS_NAME, key);
0502                 entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, key);
0503                 entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFREG);
0504                 entry.fastInsert(KIO::UDSEntry::UDS_SIZE, object.GetSize());
0505                 entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
0506                 entry.fastInsert(KIO::UDSEntry::UDS_MODIFICATION_TIME, object.GetLastModified().SecondsWithMSPrecision());
0507                 q->listEntry(entry);
0508             }
0509         }
0510
0511         const auto commonPrefixes = listObjectsOutcome.GetResult().GetCommonPrefixes();
0512         qCDebug(S3) << "Prefix" << prefix << "has" << commonPrefixes.size() << "common prefixes";
0513         for (const auto &commonPrefix : commonPrefixes) {
0514             QString subprefix = QString::fromUtf8(commonPrefix.GetPrefix().c_str(), commonPrefix.GetPrefix().size());
0515             if (subprefix.endsWith(QLatin1Char('/'))) {
0516                 subprefix.chop(1);
0517             }
0518             if (subprefix.startsWith(prefix)) {
0519                 subprefix.remove(0, prefix.length());
0520             }
0521             KIO::UDSEntry entry;
0522             entry.reserve(4);
0523             entry.fastInsert(KIO::UDSEntry::UDS_NAME, subprefix);
0524             entry.fastInsert(KIO::UDSEntry::UDS_DISPLAY_NAME, subprefix);
0525             entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0526             entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0527
0528             q->listEntry(entry);
0529         }
0530     } else {
0531         qCDebug(S3) << "Could not list prefix" << s3url.key() << " - " << listObjectsOutcome.GetError().GetMessage().c_str();
0532     }
0533 }
0534
0535 void S3Backend::listCwdEntry(CwdAccess access)
0536 {
0537     // List UDSEntry for "."
0538     KIO::UDSEntry entry;
0539     entry.reserve(4);
0540     entry.fastInsert(KIO::UDSEntry::UDS_NAME, QStringLiteral("."));
0541     entry.fastInsert(KIO::UDSEntry::UDS_FILE_TYPE, S_IFDIR);
0542     entry.fastInsert(KIO::UDSEntry::UDS_SIZE, 0);
0543     if (access == ReadOnlyCwd) {
0544         entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
0545     } else {
0546         entry.fastInsert(KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH);
0547     }
0548
0549     q->listEntry(entry);
0550 }
0551
0552 bool S3Backend::deletePrefix(const Aws::S3::S3Client &client, const S3Url &s3url, const Aws::String &prefix)
0553 {
0554     qCDebug(S3) << "Going to recursively delete prefix:" << prefix.c_str();
0555     bool outcome = false;
0556     const Aws::String bucketName = s3url.BucketName();
0557     // In order to recursively delete a folder, we need to list by prefix and manually delete each listed key.
0558     Aws::S3::Model::ListObjectsV2Request listObjectsRequest;
0559     listObjectsRequest.SetBucket(bucketName);
0560     listObjectsRequest.SetDelimiter("/");
0561     listObjectsRequest.SetPrefix(prefix);
0562     const auto listObjectsOutcome = client.ListObjectsV2(listObjectsRequest);
0563     if (listObjectsOutcome.IsSuccess()) {
0564         // TODO: handle listObjectsOutcome.GetResult().GetIsTruncated()
0565         // By default the max-keys request parameter is 1000, which is reasonable for us
0566         // since we filter the keys by the name of the folder, but it won't work
0567         // if someone has very big folders with more than 1000 files.
0568         const auto commonPrefixes = listObjectsOutcome.GetResult().GetCommonPrefixes();
0569         qCDebug(S3) << "Prefix" << prefix.c_str() << "has" << commonPrefixes.size() << "common prefixes";
0570         // Recursively delete folder children.
0571         for (const auto &commonPrefix : commonPrefixes) {
0572             const bool recursiveOutcome = deletePrefix(client, s3url, commonPrefix.GetPrefix());
0573             outcome = outcome && recursiveOutcome;
0574         }
0575         const auto objects = listObjectsOutcome.GetResult().GetContents();
0576         // Delete each file child.
0577         if (objects.size() > 0) {
0578             for (const auto &object : objects) {
0579                 Aws::S3::Model::DeleteObjectRequest request;
0580                 request.SetBucket(bucketName);
0581                 request.SetKey(object.GetKey());
0582                 auto deleteObjectOutcome = client.DeleteObject(request);
0583                 if (!deleteObjectOutcome.IsSuccess()) {
0584                     qCDebug(S3) << "Could not delete object with key:" << s3url.key() << " - " << deleteObjectOutcome.GetError().GetMessage().c_str();
0585                     outcome = false;
0586                 }
0587             }
0588         } else { // The prefix was either a file or a folder that contains 0 files.
0589             Aws::S3::Model::DeleteObjectRequest request;
0590             request.SetBucket(bucketName);
0591             request.SetKey(s3url.Key());
0592             auto deleteObjectOutcome = client.DeleteObject(request);
0593             if (!deleteObjectOutcome.IsSuccess()) {
0594                 qCDebug(S3) << "Could not delete object with key:" << s3url.key() << " - " << deleteObjectOutcome.GetError().GetMessage().c_str();
0595                 outcome = false;
0596             }
0597         }
0598         outcome = true;
0599     } else {
0600         qCDebug(S3) << "Could not list prefix:" << prefix.c_str();
0601         outcome = false;
0602     }
0603
0604     return outcome;
0605 }
0606
0607 QString S3Backend::contentType(const S3Url &s3url)
0608 {
0609     QString contentType;
0610
0611     const Aws::Client::ClientConfiguration clientConfiguration(m_configProfileName.c_str());
0612     const Aws::S3::S3Client client(clientConfiguration);
0613
0614     Aws::S3::Model::HeadObjectRequest headObjectRequest;
0615     headObjectRequest.SetBucket(s3url.BucketName());
0616     headObjectRequest.SetKey(s3url.Key());
0617
0618     auto headObjectRequestOutcome = client.HeadObject(headObjectRequest);
0619     if (headObjectRequestOutcome.IsSuccess()) {
0620         contentType = QString::fromLatin1(headObjectRequestOutcome.GetResult().GetContentType().c_str(), headObjectRequestOutcome.GetResult().GetContentType().size());
0621         qCDebug(S3) << "Key" << s3url.key() << "has Content-Type:" << contentType;
0622     } else {
0623         qCDebug(S3) << "Could not get content type for key:" << s3url.key() << " - " << headObjectRequestOutcome.GetError().GetMessage().c_str();
0624     }
0625
0626     return contentType;
0627 }