webcrawler.h

Go to the documentation of this file.

 
#ifndef WEBCRAWLER_H
#define WEBCRAWLER_H
 
#include <QNetworkReply>
#include <QQueue>
 
QT_BEGIN_NAMESPACE
class QUrl;
QT_END_NAMESPACE
 
using namespace std;
 
class WebCrawler : public QObject  {
    Q_OBJECT
    //    QThread wc_spiderThread;
public:
 
    WebCrawler (
            QQueue<QUrl> *urlQueue,
            const QUrl &startUrl,
            const QStringList &urlPatternsIncluded,
            const QStringList &urlPatternsExcluded,
            const QStringList &linkClasses,
            const int &maxNodes,
            const int &maxLinksPerPage,
            const bool &intLinks = true,
            const bool &childLinks = true,
            const bool &parentLinks = false,
            const bool &selfLinks = false,
            const bool &extLinksIncluded = false,
            const bool &extLinksCrawl = false,
            const bool &socialLinks = false,
            const int &delayBetween  = 0
            );
 
    ~WebCrawler();
 
public slots:
    void parse(QNetworkReply *reply);
    void newLink(int s, QUrl target, bool enqueue_to_frontier);
 
signals:
    void signalCreateNode(const int &no,
                          const QString &url,
                          const bool &signalMW=false);
    void signalCreateEdge (const int &source, const int &target);
    void signalStartSpider();
    void finished (QString);
 
private:
    QQueue<QUrl> *m_urlQueue;
    QMap <QUrl, int> knownUrls;
    QUrl m_initialUrl;
    int m_maxUrls;
    int m_discoveredNodes;
    int m_maxLinksPerPage;
 
    bool m_intLinks;
    bool m_childLinks;
    bool m_parentLinks;
    bool m_selfLinks ;
    bool m_extLinksIncluded;
    bool m_extLinksCrawl;
    bool m_socialLinks;
    bool m_urlIsSocial;
 
    int m_delayBetween;
 
    QStringList m_urlPatternsIncluded;
    QString urlPattern;
    QStringList m_urlPatternsExcluded;
    QStringList m_linkClasses;
    QStringList m_socialLinksExcluded;
    QStringList::const_iterator constIterator;
    bool m_urlPatternAllowed;
    bool m_urlPatternNotAllowed;
    bool m_linkClassAllowed;
};
 
 
 
#endif