webcrawler.h

Go to the documentation of this file.

/***************************************************************************
 SocNetV: Social Network Visualizer
 version: 3.1
 Written in Qt
 
 webcrawler.h - description
 -------------------
 copyright : (C) 2005-2023 by Dimitris B. Kalamaras
 project site : https://socnetv.org
 
 ***************************************************************************/
 
/*******************************************************************************
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
********************************************************************************/
 
#ifndef WEBCRAWLER_H
#define WEBCRAWLER_H
 
#include <QNetworkReply>
#include <QQueue>
 
QT_BEGIN_NAMESPACE
class QUrl;
QT_END_NAMESPACE
 
using namespace std;
 
class WebCrawler : public QObject {
 Q_OBJECT
 // QThread wc_spiderThread;
public:
 
 WebCrawler (
 QQueue<QUrl> *urlQueue,
 const QUrl &startUrl,
 const QStringList &urlPatternsIncluded,
 const QStringList &urlPatternsExcluded,
 const QStringList &linkClasses,
 const int &maxNodes,
 const int &maxLinksPerPage,
 const bool &intLinks = true,
 const bool &childLinks = true,
 const bool &parentLinks = false,
 const bool &selfLinks = false,
 const bool &extLinksIncluded = false,
 const bool &extLinksCrawl = false,
 const bool &socialLinks = false,
 const int &delayBetween = 0
 );
 
 ~WebCrawler();
 
public slots:
 void parse(QNetworkReply *reply);
 void newLink(int s, QUrl target, bool enqueue_to_frontier);
 
signals:
 void signalCreateNode(const int &no,
 const QString &url,
 const bool &signalMW=false);
 void signalCreateEdge (const int &source, const int &target);
 void signalStartSpider();
 void finished (QString);
 
private:
 QQueue<QUrl> *m_urlQueue;
 QMap <QUrl, int> knownUrls;
 QUrl m_initialUrl;
 int m_maxUrls;
 int m_discoveredNodes;
 int m_maxLinksPerPage;
 
 bool m_intLinks;
 bool m_childLinks;
 bool m_parentLinks;
 bool m_selfLinks ;
 bool m_extLinksIncluded;
 bool m_extLinksCrawl;
 bool m_socialLinks;
 bool m_urlIsSocial;
 
 int m_delayBetween;
 
 QStringList m_urlPatternsIncluded;
 QString urlPattern;
 QStringList m_urlPatternsExcluded;
 QStringList m_linkClasses;
 QStringList m_socialLinksExcluded;
 QStringList::const_iterator constIterator;
 bool m_urlPatternAllowed;
 bool m_urlPatternNotAllowed;
 bool m_linkClassAllowed;
};
 
 
 
#endif