src/jobs/audiolevelstask.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2013-2021 Meltytech LLC
0003     SPDX-FileCopyrightText: 2021 Jean-Baptiste Mardelle <jb@kdenlive.org>
0004
0005     SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
0006 */
0007
0008 #include "audiolevelstask.h"
0009 #include "audio/audioStreamInfo.h"
0010 #include "bin/projectclip.h"
0011 #include "bin/projectitemmodel.h"
0012 #include "core.h"
0013
0014 #include <KLocalizedString>
0015 #include <KMessageWidget>
0016 #include <QElapsedTimer>
0017 #include <QFile>
0018 #include <QImage>
0019 #include <QList>
0020 #include <QMutex>
0021 #include <QRgb>
0022 #include <QString>
0023 #include <QThreadPool>
0024 #include <QTime>
0025 #include <QVariantList>
0026
0027 static QList<AudioLevelsTask *> tasksList;
0028 static QMutex tasksListMutex;
0029
0030 static void deleteQVariantList(QVector<uint8_t> *list)
0031 {
0032     delete list;
0033 }
0034
0035 AudioLevelsTask::AudioLevelsTask(const ObjectId &owner, QObject *object)
0036     : AbstractTask(owner, AbstractTask::AUDIOTHUMBJOB, object)
0037 {
0038     m_description = i18n("Audio thumbs");
0039 }
0040
0041 void AudioLevelsTask::start(const ObjectId &owner, QObject *object, bool force)
0042 {
0043     // See if there is already a task for this MLT service and resource.
0044     if (pCore->taskManager.hasPendingJob(owner, AbstractTask::AUDIOTHUMBJOB)) {
0045         qDebug() << "AUDIO LEVELS TASK STARTED TWICE!!!!";
0046         return;
0047     }
0048     AudioLevelsTask *task = new AudioLevelsTask(owner, object);
0049     // Otherwise, start a new audio levels generation thread.
0050     task->m_isForce = force;
0051     pCore->taskManager.startTask(owner.itemId, task);
0052 }
0053
0054 void AudioLevelsTask::run()
0055 {
0056     AbstractTaskDone whenFinished(m_owner.itemId, this);
0057     if (m_isCanceled || pCore->taskManager.isBlocked()) {
0058         return;
0059     }
0060     QMutexLocker lock(&m_runMutex);
0061     m_running = true;
0062     // 2 channels interleaved of uchar values
0063     auto binClip = pCore->projectItemModel()->getClipByBinID(QString::number(m_owner.itemId));
0064     if (binClip == nullptr) {
0065         // Clip was deleted
0066         return;
0067     }
0068     if (binClip->audioChannels() == 0 || binClip->audioThumbCreated()) {
0069         // nothing to do
0070         return;
0071     }
0072     std::shared_ptr<Mlt::Producer> producer = binClip->originalProducer();
0073     if ((producer == nullptr) || !producer->is_valid()) {
0074         QMetaObject::invokeMethod(pCore.get(), "displayBinMessage", Qt::QueuedConnection,
0075                                   Q_ARG(QString, i18n("Audio thumbs: cannot open file %1", QFileInfo(binClip->url()).fileName())),
0076                                   Q_ARG(int, int(KMessageWidget::Warning)));
0077         return;
0078     }
0079     int lengthInFrames = producer->get_length(); // Multiply this if we want more than 1 sample per frame
0080     if (lengthInFrames == INT_MAX || lengthInFrames == 0) {
0081         // This is a broken file or live feed, don't attempt to generate audio thumbnails
0082         QMetaObject::invokeMethod(pCore.get(), "displayBinMessage", Qt::QueuedConnection,
0083                                   Q_ARG(QString, i18n("Audio thumbs: unknown file length for %1", QFileInfo(binClip->url()).fileName())),
0084                                   Q_ARG(int, int(KMessageWidget::Warning)));
0085         return;
0086     }
0087     QString service = producer->get("mlt_service");
0088     if (service == QLatin1String("avformat-novalidate")) {
0089         service = QStringLiteral("avformat");
0090     } else if (service.startsWith(QLatin1String("xml"))) {
0091         service = QStringLiteral("xml-nogl");
0092     }
0093     const QString res = qstrdup(producer->get("resource"));
0094     producer.reset();
0095     int frequency = binClip->audioInfo()->samplingRate();
0096     frequency = frequency <= 0 ? 48000 : frequency;
0097
0098     int channels = binClip->audioInfo()->channels();
0099     channels = channels <= 0 ? 2 : channels;
0100
0101     QMap<int, QString> streams = binClip->audioInfo()->streams();
0102     QMap<int, int> audioChannels = binClip->audioInfo()->streamChannels();
0103     QMapIterator<int, QString> st(streams);
0104     bool audioCreated = false;
0105     int streamIndex = -1;
0106     while (st.hasNext() && !m_isCanceled) {
0107         st.next();
0108         int stream = st.key();
0109         if (audioChannels.contains(stream)) {
0110             channels = audioChannels.value(stream);
0111         }
0112         streamIndex++;
0113         // Generate one thumb per stream
0114         QString cachePath = binClip->getAudioThumbPath(stream);
0115         QVector<uint8_t> mltLevels;
0116         if (!m_isForce && QFile::exists(cachePath)) {
0117             // Audio thumb already exists
0118             QImage image(cachePath);
0119             if (!m_isCanceled && !image.isNull()) {
0120                 // convert cached image
0121                 int n = image.width() * image.height();
0122                 for (int i = 0; n > 1 && i < n; i++) {
0123                     QRgb p = image.pixel(i / channels, i % channels);
0124                     mltLevels << qRed(p);
0125                     mltLevels << qGreen(p);
0126                     mltLevels << qBlue(p);
0127                     mltLevels << qAlpha(p);
0128                 }
0129                 if (mltLevels.size() > 0) {
0130                     QVector<uint8_t> *levelsCopy = new QVector<uint8_t>(mltLevels);
0131                     producer = binClip->originalProducer();
0132                     producer->lock();
0133                     QString key = QString("_kdenlive:audio%1").arg(stream);
0134                     producer->set(key.toUtf8().constData(), levelsCopy, 0, (mlt_destructor)deleteQVariantList);
0135                     producer->unlock();
0136                     producer.reset();
0137                     continue;
0138                 }
0139             }
0140         }
0141
0142         Mlt::Producer *aProd = new Mlt::Producer(pCore->getProjectProfile(), service.toUtf8().constData(), res.toUtf8().constData());
0143         if (!aProd->is_valid()) {
0144             QMetaObject::invokeMethod(pCore.get(), "displayBinMessage", Qt::QueuedConnection, Q_ARG(QString, i18n("Audio thumbs: cannot open file %1", res)),
0145                                       Q_ARG(int, int(KMessageWidget::Warning)));
0146             delete aProd;
0147             return;
0148         }
0149         aProd->set("video_index", -1);
0150         aProd->set("audio_index", stream);
0151         aProd->set("vstream", -1);
0152         aProd->set("astream", streamIndex);
0153         Mlt::Filter chans(pCore->getProjectProfile(), "audiochannels");
0154         Mlt::Filter converter(pCore->getProjectProfile(), "audioconvert");
0155         Mlt::Filter levels(pCore->getProjectProfile(), "audiolevel");
0156         aProd->attach(chans);
0157         aProd->attach(converter);
0158         aProd->attach(levels);
0159         std::unique_ptr<Mlt::Producer> audioProducer;
0160         audioProducer.reset(aProd);
0161
0162         double framesPerSecond = audioProducer->get_fps();
0163         mlt_audio_format audioFormat = mlt_audio_s16;
0164         QStringList keys;
0165         keys.reserve(channels);
0166         for (int i = 0; i < channels; i++) {
0167             keys << "meta.media.audio_level." + QString::number(i);
0168         }
0169         uint maxLevel = 1;
0170         QElapsedTimer updateTime;
0171         updateTime.start();
0172         for (int z = 0; z < lengthInFrames && !m_isCanceled; ++z) {
0173             int val = int(100.0 * z / lengthInFrames);
0174             if (m_progress != val) {
0175                 m_progress = val;
0176                 QMetaObject::invokeMethod(m_object, "updateJobProgress");
0177             }
0178             QScopedPointer<Mlt::Frame> mltFrame(audioProducer->get_frame());
0179             if ((mltFrame != nullptr) && mltFrame->is_valid() && (mltFrame->get_int("test_audio") == 0)) {
0180                 int samples = mlt_audio_calculate_frame_samples(float(framesPerSecond), frequency, z);
0181                 mltFrame->get_audio(audioFormat, frequency, channels, samples);
0182                 for (int channel = 0; channel < channels; ++channel) {
0183                     uint lev = 256 * qMin(mltFrame->get_double(keys.at(channel).toUtf8().constData()) * 0.9, 1.0);
0184                     mltLevels << lev;
0185                     // double lev = mltFrame->get_double(keys.at(channel).toUtf8().constData());
0186                     // mltLevels << lev;
0187                     maxLevel = qMax(lev, maxLevel);
0188                 }
0189             } else if (!mltLevels.isEmpty()) {
0190                 for (int channel = 0; channel < channels; channel++) {
0191                     mltLevels << mltLevels.last();
0192                 }
0193             }
0194             // Incrementally update the audio levels every 3 seconds.
0195             if (updateTime.elapsed() > 3000 && !m_isCanceled) {
0196                 updateTime.restart();
0197                 QVector<uint8_t> *levelsCopy = new QVector<uint8_t>(mltLevels);
0198                 producer = binClip->originalProducer();
0199                 producer->lock();
0200                 QString key = QString("_kdenlive:audio%1").arg(stream);
0201                 producer->set(key.toUtf8().constData(), levelsCopy, 0, (mlt_destructor)deleteQVariantList);
0202                 producer->unlock();
0203                 producer.reset();
0204                 QMetaObject::invokeMethod(m_object, "updateAudioThumbnail", Q_ARG(bool, false));
0205             }
0206         }
0207
0208         /*// Normalize
0209         for (double &v : mltLevels) {
0210             m_audioLevels << uchar(255 * v / maxLevel);
0211         }*/
0212         if (m_isCanceled) {
0213             mltLevels.clear();
0214             m_progress = 100;
0215             QMetaObject::invokeMethod(m_object, "updateJobProgress");
0216         }
0217         if (mltLevels.size() > 0) {
0218             QVector<uint8_t> *levelsCopy = new QVector<uint8_t>(mltLevels);
0219             producer = binClip->originalProducer();
0220             producer->lock();
0221             QString key = QString("_kdenlive:audio%1").arg(stream);
0222             QString key2 = QString("kdenlive:audio_max%1").arg(stream);
0223             producer->set(key2.toUtf8().constData(), int(maxLevel));
0224             producer->set(key.toUtf8().constData(), levelsCopy, 0, (mlt_destructor)deleteQVariantList);
0225             producer->unlock();
0226             producer.reset();
0227             // qDebug()<<"=== FINISHED PRODUCING AUDIO FOR: "<<key<<", SIZE: "<<levelsCopy->size();
0228             m_progress = 100;
0229             QMetaObject::invokeMethod(m_object, "updateJobProgress");
0230             // Put into an image for caching.
0231             int count = mltLevels.size();
0232             QImage image((count + 3) / 4 / channels, channels, QImage::Format_ARGB32);
0233             int n = image.width() * image.height();
0234             for (int i = 0; i < n; i++) {
0235                 QRgb p;
0236                 if ((4 * i + 3) < count) {
0237                     p = qRgba(mltLevels.at(4 * i), mltLevels.at(4 * i + 1), mltLevels.at(4 * i + 2), mltLevels.at(4 * i + 3));
0238                 } else {
0239                     int last = mltLevels.last();
0240                     int r = (4 * i + 0) < count ? mltLevels.at(4 * i + 0) : last;
0241                     int g = (4 * i + 1) < count ? mltLevels.at(4 * i + 1) : last;
0242                     int b = (4 * i + 2) < count ? mltLevels.at(4 * i + 2) : last;
0243                     int a = last;
0244                     p = qRgba(r, g, b, a);
0245                 }
0246                 image.setPixel(i / channels, i % channels, p);
0247             }
0248             image.save(cachePath);
0249             audioCreated = true;
0250             QMetaObject::invokeMethod(m_object, "updateAudioThumbnail", Q_ARG(bool, false));
0251         }
0252     }
0253     if (!audioCreated && !m_isCanceled) {
0254         // Audio was cached, ensure the bin thumbnail is loaded
0255         QMetaObject::invokeMethod(m_object, "updateAudioThumbnail", Q_ARG(bool, true));
0256     }
0257     QMetaObject::invokeMethod(m_object, "updateJobProgress");
0258 }