From 82d0d6fcc96268a6931f01fcbab7c39b4ea71e9f Mon Sep 17 00:00:00 2001 From: Joe Ferris Date: Mon, 24 Nov 2014 09:52:34 -0500 Subject: [PATCH] Extract handle classes from NetworkAccessManager NetworkAccessManager was getting pretty complex, and adding something that modifies or intercepts requests is becoming a common change. This change introduces a chain of handlers which can modify a request or return a response of their own. To start with, this extracts the concerns of fixing missing content types, setting custom headers, and intercepting blacklisted requests. --- src/BlacklistedRequestHandler.cpp | 48 ++++++++++++ src/BlacklistedRequestHandler.h | 20 +++++ src/CustomHeadersRequestHandler.cpp | 37 +++++++++ src/CustomHeadersRequestHandler.h | 21 ++++++ src/Header.cpp | 3 +- src/MissingContentHeaderRequestHandler.cpp | 23 ++++++ src/MissingContentHeaderRequestHandler.h | 15 ++++ src/NetworkAccessManager.cpp | 88 +++++++++------------- src/NetworkAccessManager.h | 23 +++--- src/NetworkRequestFactory.cpp | 15 ++++ src/NetworkRequestFactory.h | 12 +++ src/NoOpReply.cpp | 2 +- src/NoOpReply.h | 2 +- src/RequestHandler.cpp | 4 + src/RequestHandler.h | 25 ++++++ src/SetUrlBlacklist.cpp | 3 +- src/SetUrlBlacklist.h | 2 - src/WebPageManager.cpp | 25 +++++- src/WebPageManager.h | 6 ++ src/webkit_server.pro | 14 +++- 20 files changed, 315 insertions(+), 73 deletions(-) create mode 100644 src/BlacklistedRequestHandler.cpp create mode 100644 src/BlacklistedRequestHandler.h create mode 100644 src/CustomHeadersRequestHandler.cpp create mode 100644 src/CustomHeadersRequestHandler.h create mode 100644 src/MissingContentHeaderRequestHandler.cpp create mode 100644 src/MissingContentHeaderRequestHandler.h create mode 100644 src/NetworkRequestFactory.cpp create mode 100644 src/NetworkRequestFactory.h create mode 100644 src/RequestHandler.cpp create mode 100644 src/RequestHandler.h diff --git a/src/BlacklistedRequestHandler.cpp b/src/BlacklistedRequestHandler.cpp new file mode 100644 index 0000000..b0f7d2f --- /dev/null +++ b/src/BlacklistedRequestHandler.cpp @@ -0,0 +1,48 @@ +#include "BlacklistedRequestHandler.h" +#include "NetworkReplyProxy.h" +#include "NoOpReply.h" + +BlacklistedRequestHandler::BlacklistedRequestHandler( + RequestHandler *next, + QObject *parent +) : RequestHandler(parent) { + m_next = next; +} + +QNetworkReply* BlacklistedRequestHandler::handleRequest( + NetworkAccessManager *manager, + QNetworkAccessManager::Operation operation, + QNetworkRequest &request, + QIODevice *outgoingData +) { + if (this->isBlacklisted(request.url())) { + return new NetworkReplyProxy(new NoOpReply(request), this); + } else { + return m_next->handleRequest(manager, operation, request, outgoingData); + } +} + +void BlacklistedRequestHandler::setUrlBlacklist(QStringList urlBlacklist) { + m_urlBlacklist.clear(); + + QStringListIterator iter(urlBlacklist); + while (iter.hasNext()) { + m_urlBlacklist << iter.next(); + } +} + +bool BlacklistedRequestHandler::isBlacklisted(QUrl url) { + QString urlString = url.toString(); + QStringListIterator iter(m_urlBlacklist); + + while (iter.hasNext()) { + QRegExp blacklisted = QRegExp(iter.next()); + blacklisted.setPatternSyntax(QRegExp::Wildcard); + + if(urlString.contains(blacklisted)) { + return true; + } + } + + return false; +} diff --git a/src/BlacklistedRequestHandler.h b/src/BlacklistedRequestHandler.h new file mode 100644 index 0000000..20d991c --- /dev/null +++ b/src/BlacklistedRequestHandler.h @@ -0,0 +1,20 @@ +#include + +#include "RequestHandler.h" + +class BlacklistedRequestHandler : public RequestHandler { + public: + BlacklistedRequestHandler(RequestHandler *next, QObject *parent = 0); + virtual QNetworkReply* handleRequest( + NetworkAccessManager *, + QNetworkAccessManager::Operation, + QNetworkRequest &, + QIODevice * + ); + void setUrlBlacklist(QStringList urlBlacklist); + + private: + RequestHandler *m_next; + QStringList m_urlBlacklist; + bool isBlacklisted(QUrl url); +}; diff --git a/src/CustomHeadersRequestHandler.cpp b/src/CustomHeadersRequestHandler.cpp new file mode 100644 index 0000000..8beb159 --- /dev/null +++ b/src/CustomHeadersRequestHandler.cpp @@ -0,0 +1,37 @@ +#include "CustomHeadersRequestHandler.h" +#include "NetworkReplyProxy.h" +#include "NoOpReply.h" + +CustomHeadersRequestHandler::CustomHeadersRequestHandler( + RequestHandler *next, + QObject *parent +) : RequestHandler(parent) { + m_next = next; +} + +QNetworkReply* CustomHeadersRequestHandler::handleRequest( + NetworkAccessManager *manager, + QNetworkAccessManager::Operation operation, + QNetworkRequest &request, + QIODevice *outgoingData +) { + Q_UNUSED(manager) + Q_UNUSED(operation) + Q_UNUSED(outgoingData) + + QHashIterator item(m_headers); + while (item.hasNext()) { + item.next(); + request.setRawHeader(item.key().toLatin1(), item.value().toLatin1()); + } + + return m_next->handleRequest(manager, operation, request, outgoingData); +} + +void CustomHeadersRequestHandler::addHeader(QString key, QString value) { + m_headers.insert(key, value); +} + +void CustomHeadersRequestHandler::reset() { + m_headers.clear(); +} diff --git a/src/CustomHeadersRequestHandler.h b/src/CustomHeadersRequestHandler.h new file mode 100644 index 0000000..b187f5a --- /dev/null +++ b/src/CustomHeadersRequestHandler.h @@ -0,0 +1,21 @@ +#include +#include + +#include "RequestHandler.h" + +class CustomHeadersRequestHandler : public RequestHandler { + public: + CustomHeadersRequestHandler(RequestHandler *next, QObject *parent = 0); + virtual QNetworkReply* handleRequest( + NetworkAccessManager *, + QNetworkAccessManager::Operation, + QNetworkRequest &, + QIODevice * + ); + void addHeader(QString, QString); + virtual void reset(); + + private: + RequestHandler *m_next; + QHash m_headers; +}; diff --git a/src/Header.cpp b/src/Header.cpp index 2d2eae8..4921ed8 100644 --- a/src/Header.cpp +++ b/src/Header.cpp @@ -9,11 +9,10 @@ Header::Header(WebPageManager *manager, QStringList &arguments, QObject *parent) void Header::start() { QString key = arguments()[0]; QString value = arguments()[1]; - NetworkAccessManager* networkAccessManager = manager()->networkAccessManager(); if (key.toLower().replace("-", "_") == "user_agent") { page()->setUserAgent(value); } else { - networkAccessManager->addHeader(key, value); + manager()->addHeader(key, value); } finish(true); } diff --git a/src/MissingContentHeaderRequestHandler.cpp b/src/MissingContentHeaderRequestHandler.cpp new file mode 100644 index 0000000..c450c0e --- /dev/null +++ b/src/MissingContentHeaderRequestHandler.cpp @@ -0,0 +1,23 @@ +#include "MissingContentHeaderRequestHandler.h" +#include "NetworkReplyProxy.h" +#include "NoOpReply.h" + +MissingContentHeaderRequestHandler::MissingContentHeaderRequestHandler( + RequestHandler *next, + QObject *parent +) : RequestHandler(parent) { + m_next = next; +} + +QNetworkReply* MissingContentHeaderRequestHandler::handleRequest( + NetworkAccessManager *manager, + QNetworkAccessManager::Operation operation, + QNetworkRequest &request, + QIODevice *outgoingData +) { + if (operation != QNetworkAccessManager::PostOperation && operation != QNetworkAccessManager::PutOperation) { + request.setHeader(QNetworkRequest::ContentTypeHeader, QVariant()); + } + + return m_next->handleRequest(manager, operation, request, outgoingData); +} diff --git a/src/MissingContentHeaderRequestHandler.h b/src/MissingContentHeaderRequestHandler.h new file mode 100644 index 0000000..14ffcf7 --- /dev/null +++ b/src/MissingContentHeaderRequestHandler.h @@ -0,0 +1,15 @@ +#include "RequestHandler.h" + +class MissingContentHeaderRequestHandler : public RequestHandler { + public: + MissingContentHeaderRequestHandler(RequestHandler *next, QObject *parent = 0); + virtual QNetworkReply* handleRequest( + NetworkAccessManager *, + QNetworkAccessManager::Operation, + QNetworkRequest &, + QIODevice * + ); + + private: + RequestHandler *m_next; +}; diff --git a/src/NetworkAccessManager.cpp b/src/NetworkAccessManager.cpp index ed88399..1e45f0c 100644 --- a/src/NetworkAccessManager.cpp +++ b/src/NetworkAccessManager.cpp @@ -1,34 +1,46 @@ #include "NetworkAccessManager.h" #include "WebPage.h" -#include -#include -#include "NoOpReply.h" #include "NetworkReplyProxy.h" +#include "RequestHandler.h" -NetworkAccessManager::NetworkAccessManager(QObject *parent):QNetworkAccessManager(parent) { +NetworkAccessManager::NetworkAccessManager( + RequestHandler * requestHandler, + QObject *parent +) : QNetworkAccessManager(parent) { + m_requestHandler = requestHandler; connect(this, SIGNAL(authenticationRequired(QNetworkReply*,QAuthenticator*)), SLOT(provideAuthentication(QNetworkReply*,QAuthenticator*))); connect(this, SIGNAL(finished(QNetworkReply *)), this, SLOT(finished(QNetworkReply *))); disableKeyChainLookup(); } -QNetworkReply* NetworkAccessManager::createRequest(QNetworkAccessManager::Operation operation, const QNetworkRequest &request, QIODevice * outgoingData = 0) { - QNetworkRequest new_request(request); - QByteArray url = new_request.url().toEncoded(); - if (this->isBlacklisted(new_request.url())) { - return new NetworkReplyProxy(new NoOpReply(new_request), this); - } else { - if (operation != QNetworkAccessManager::PostOperation && operation != QNetworkAccessManager::PutOperation) { - new_request.setHeader(QNetworkRequest::ContentTypeHeader, QVariant()); - } - QHashIterator item(m_headers); - while (item.hasNext()) { - item.next(); - new_request.setRawHeader(item.key().toLatin1(), item.value().toLatin1()); - } - QNetworkReply *reply = new NetworkReplyProxy(QNetworkAccessManager::createRequest(operation, new_request, outgoingData), this); - emit requestCreated(url, reply); - return reply; - } +QNetworkReply* NetworkAccessManager::sendRequest( + QNetworkAccessManager::Operation operation, + const QNetworkRequest &request, + QIODevice * outgoingData +) { + QNetworkReply *reply = new NetworkReplyProxy( + QNetworkAccessManager::createRequest(operation, + request, + outgoingData + ), + this + ); + + QByteArray url = reply->request().url().toEncoded(); + emit requestCreated(url, reply); + + return reply; +} + +QNetworkReply* NetworkAccessManager::createRequest( + QNetworkAccessManager::Operation operation, + const QNetworkRequest &unsafeRequest, + QIODevice * outgoingData = 0 +) { + QNetworkRequest request(unsafeRequest); + QNetworkReply *reply = + m_requestHandler->handleRequest(this, operation, request, outgoingData); + return reply; }; void NetworkAccessManager::finished(QNetworkReply *reply) { @@ -43,12 +55,7 @@ void NetworkAccessManager::finished(QNetworkReply *reply) { } } -void NetworkAccessManager::addHeader(QString key, QString value) { - m_headers.insert(key, value); -} - void NetworkAccessManager::reset() { - m_headers.clear(); m_userName = QString(); m_password = QString(); } @@ -63,37 +70,12 @@ void NetworkAccessManager::setPassword(const QString &password) { void NetworkAccessManager::provideAuthentication(QNetworkReply *reply, QAuthenticator *authenticator) { Q_UNUSED(reply); - if (m_userName != authenticator->user()) + if (m_userName != authenticator->user()) authenticator->setUser(m_userName); if (m_password != authenticator->password()) authenticator->setPassword(m_password); } -void NetworkAccessManager::setUrlBlacklist(QStringList urlBlacklist) { - m_urlBlacklist.clear(); - - QStringListIterator iter(urlBlacklist); - while (iter.hasNext()) { - m_urlBlacklist << iter.next(); - } -}; - -bool NetworkAccessManager::isBlacklisted(QUrl url) { - QString urlString = url.toString(); - QStringListIterator iter(m_urlBlacklist); - - while (iter.hasNext()) { - QRegExp blacklisted = QRegExp(iter.next()); - blacklisted.setPatternSyntax(QRegExp::Wildcard); - - if(urlString.contains(blacklisted)) { - return true; - } - } - - return false; -}; - /* * This is a workaround for a Qt 5/OS X bug: * https://bugreports.qt-project.org/browse/QTBUG-30434 diff --git a/src/NetworkAccessManager.h b/src/NetworkAccessManager.h index 80cff09..e077861 100644 --- a/src/NetworkAccessManager.h +++ b/src/NetworkAccessManager.h @@ -3,32 +3,37 @@ #include #include #include -#include + +class RequestHandler; class NetworkAccessManager : public QNetworkAccessManager { - Q_OBJECT public: - NetworkAccessManager(QObject *parent = 0); - void addHeader(QString key, QString value); + NetworkAccessManager(RequestHandler *, QObject *parent = 0); void reset(); void setUserName(const QString &userName); void setPassword(const QString &password); - void setUrlBlacklist(QStringList urlBlacklist); + QNetworkReply* sendRequest( + QNetworkAccessManager::Operation, + const QNetworkRequest &, + QIODevice * + ); protected: - QNetworkReply* createRequest(QNetworkAccessManager::Operation op, const QNetworkRequest &req, QIODevice * outgoingData); + QNetworkReply* createRequest( + QNetworkAccessManager::Operation, + const QNetworkRequest &, + QIODevice * + ); QString m_userName; QString m_password; - QStringList m_urlBlacklist; private: void disableKeyChainLookup(); - QHash m_headers; - bool isBlacklisted(QUrl url); QHash m_redirectMappings; + RequestHandler * m_requestHandler; private slots: void provideAuthentication(QNetworkReply *reply, QAuthenticator *authenticator); diff --git a/src/NetworkRequestFactory.cpp b/src/NetworkRequestFactory.cpp new file mode 100644 index 0000000..9621403 --- /dev/null +++ b/src/NetworkRequestFactory.cpp @@ -0,0 +1,15 @@ +#include "NetworkRequestFactory.h" +#include "NetworkAccessManager.h" + +NetworkRequestFactory::NetworkRequestFactory(QObject *parent) : + RequestHandler(parent) { +} + +QNetworkReply* NetworkRequestFactory::handleRequest( + NetworkAccessManager *manager, + QNetworkAccessManager::Operation operation, + QNetworkRequest &request, + QIODevice *outgoingData +) { + return manager->sendRequest(operation, request, outgoingData); +} diff --git a/src/NetworkRequestFactory.h b/src/NetworkRequestFactory.h new file mode 100644 index 0000000..5658a8d --- /dev/null +++ b/src/NetworkRequestFactory.h @@ -0,0 +1,12 @@ +#include "RequestHandler.h" + +class NetworkRequestFactory : public RequestHandler { + public: + NetworkRequestFactory(QObject *parent = 0); + virtual QNetworkReply* handleRequest( + NetworkAccessManager *, + QNetworkAccessManager::Operation, + QNetworkRequest &, + QIODevice * + ); +}; diff --git a/src/NoOpReply.cpp b/src/NoOpReply.cpp index 9cd069d..d9518cc 100644 --- a/src/NoOpReply.cpp +++ b/src/NoOpReply.cpp @@ -1,7 +1,7 @@ #include #include "NoOpReply.h" -NoOpReply::NoOpReply(QNetworkRequest &request, QObject *parent) : QNetworkReply(parent) { +NoOpReply::NoOpReply(const QNetworkRequest &request, QObject *parent) : QNetworkReply(parent) { open(ReadOnly | Unbuffered); setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 200); setHeader(QNetworkRequest::ContentLengthHeader, QVariant(0)); diff --git a/src/NoOpReply.h b/src/NoOpReply.h index 123c8eb..4d0d1f6 100644 --- a/src/NoOpReply.h +++ b/src/NoOpReply.h @@ -5,7 +5,7 @@ class NoOpReply : public QNetworkReply { Q_OBJECT public: - NoOpReply(QNetworkRequest &request, QObject *parent = 0); + NoOpReply(const QNetworkRequest &request, QObject *parent = 0); void abort(); qint64 bytesAvailable() const; diff --git a/src/RequestHandler.cpp b/src/RequestHandler.cpp new file mode 100644 index 0000000..a8f1ced --- /dev/null +++ b/src/RequestHandler.cpp @@ -0,0 +1,4 @@ +#include "RequestHandler.h" + +RequestHandler::RequestHandler(QObject *parent) : QObject(parent) { +} diff --git a/src/RequestHandler.h b/src/RequestHandler.h new file mode 100644 index 0000000..f220ba0 --- /dev/null +++ b/src/RequestHandler.h @@ -0,0 +1,25 @@ +#ifndef __REQUESTHANDLER_H +#define __REQUESTHANDLER_H + +#include +#include +#include +#include + +class NetworkAccessManager; + +class RequestHandler : public QObject { + Q_OBJECT + + public: + RequestHandler(QObject *parent = 0); + + virtual QNetworkReply* handleRequest( + NetworkAccessManager *, + QNetworkAccessManager::Operation, + QNetworkRequest &, + QIODevice * + ) = 0; +}; + +#endif diff --git a/src/SetUrlBlacklist.cpp b/src/SetUrlBlacklist.cpp index be720b9..c0cae3c 100644 --- a/src/SetUrlBlacklist.cpp +++ b/src/SetUrlBlacklist.cpp @@ -8,8 +8,7 @@ SetUrlBlacklist::SetUrlBlacklist(WebPageManager *manager, QStringList &arguments } void SetUrlBlacklist::start() { - NetworkAccessManager* networkAccessManager = manager()->networkAccessManager(); - networkAccessManager->setUrlBlacklist(arguments()); + manager()->setUrlBlacklist(arguments()); finish(true); } diff --git a/src/SetUrlBlacklist.h b/src/SetUrlBlacklist.h index 4673876..7a52f0d 100644 --- a/src/SetUrlBlacklist.h +++ b/src/SetUrlBlacklist.h @@ -1,4 +1,3 @@ - #include "SocketCommand.h" class SetUrlBlacklist : public SocketCommand { @@ -8,4 +7,3 @@ class SetUrlBlacklist : public SocketCommand { SetUrlBlacklist(WebPageManager *manager, QStringList &arguments, QObject *parent = 0); virtual void start(); }; - diff --git a/src/WebPageManager.cpp b/src/WebPageManager.cpp index 4851521..577e486 100644 --- a/src/WebPageManager.cpp +++ b/src/WebPageManager.cpp @@ -2,6 +2,10 @@ #include "WebPage.h" #include "NetworkCookieJar.h" #include "NetworkAccessManager.h" +#include "BlacklistedRequestHandler.h" +#include "CustomHeadersRequestHandler.h" +#include "MissingContentHeaderRequestHandler.h" +#include "NetworkRequestFactory.h" WebPageManager::WebPageManager(QObject *parent) : QObject(parent) { m_ignoreSslErrors = false; @@ -10,7 +14,17 @@ WebPageManager::WebPageManager(QObject *parent) : QObject(parent) { m_loggingEnabled = false; m_ignoredOutput = new QFile(this); m_timeout = -1; - m_networkAccessManager = new NetworkAccessManager(this); + m_customHeadersRequestHandler = new CustomHeadersRequestHandler( + new MissingContentHeaderRequestHandler( + new NetworkRequestFactory(this), + this + ), + this + ); + m_blacklistedRequestHandler = + new BlacklistedRequestHandler(m_customHeadersRequestHandler, this); + m_networkAccessManager = + new NetworkAccessManager(m_blacklistedRequestHandler, this); m_networkAccessManager->setCookieJar(m_cookieJar); createPage()->setFocus(); } @@ -119,6 +133,7 @@ void WebPageManager::reset() { m_timeout = -1; m_cookieJar->clearCookies(); m_networkAccessManager->reset(); + m_customHeadersRequestHandler->reset(); m_currentPage->resetLocalStorage(); while (!m_pages.isEmpty()) { WebPage *page = m_pages.takeFirst(); @@ -157,3 +172,11 @@ QDebug WebPageManager::logger() const { void WebPageManager::enableLogging() { m_loggingEnabled = true; } + +void WebPageManager::setUrlBlacklist(const QStringList &urls) { + m_blacklistedRequestHandler->setUrlBlacklist(urls); +} + +void WebPageManager::addHeader(QString key, QString value) { + m_customHeadersRequestHandler->addHeader(key, value); +} diff --git a/src/WebPageManager.h b/src/WebPageManager.h index c7d8560..2d9f411 100644 --- a/src/WebPageManager.h +++ b/src/WebPageManager.h @@ -10,6 +10,8 @@ class WebPage; class NetworkCookieJar; class NetworkAccessManager; +class BlacklistedRequestHandler; +class CustomHeadersRequestHandler; class WebPageManager : public QObject { Q_OBJECT @@ -33,6 +35,8 @@ class WebPageManager : public QObject { void enableLogging(); void replyFinished(QNetworkReply *reply); NetworkAccessManager *networkAccessManager(); + void setUrlBlacklist(const QStringList &); + void addHeader(QString, QString); public slots: void emitLoadStarted(); @@ -58,6 +62,8 @@ class WebPageManager : public QObject { QFile *m_ignoredOutput; int m_timeout; NetworkAccessManager *m_networkAccessManager; + BlacklistedRequestHandler *m_blacklistedRequestHandler; + CustomHeadersRequestHandler *m_customHeadersRequestHandler; }; #endif // _WEBPAGEMANAGER_H diff --git a/src/webkit_server.pro b/src/webkit_server.pro index bfac67f..530e35d 100644 --- a/src/webkit_server.pro +++ b/src/webkit_server.pro @@ -78,7 +78,12 @@ HEADERS = \ FindXpath.h \ NetworkReplyProxy.h \ IgnoreDebugOutput.h \ - StdinNotifier.h + StdinNotifier.h \ + RequestHandler.h \ + BlacklistedRequestHandler.h \ + MissingContentHeaderRequestHandler.h \ + CustomHeadersRequestHandler.h \ + NetworkRequestFactory.h SOURCES = \ FindModal.cpp \ @@ -153,7 +158,12 @@ SOURCES = \ FindXpath.cpp \ NetworkReplyProxy.cpp \ IgnoreDebugOutput.cpp \ - StdinNotifier.cpp + StdinNotifier.cpp \ + RequestHandler.cpp \ + BlacklistedRequestHandler.cpp \ + MissingContentHeaderRequestHandler.cpp \ + CustomHeadersRequestHandler.cpp \ + NetworkRequestFactory.cpp RESOURCES = webkit_server.qrc QT += network