VenueScan Lead Runner - Personal or Commercial

1// VenueScan Lead Runner - Personal or Commercial — lead generation module
2//
3// A crawler that finds the people behind a website: fetch a page, tokenise the
4// HTML properly, pull out the emails, phone numbers and social profiles, and
5// append them to a deduplicated JSONL store where every record remembers the
6// URL it came from.
7//
8//   leadgen.scrape    one URL  -> leads
9//   leadgen.crawl     a seed   -> leads, breadth-first to a depth limit
10//   leadgen.enrich    a site   -> leads, but only from contact/about/team pages
11//   leadgen.export    the store-> CSV (RFC 4180)
12//   leadgen.campaign  a named set of seeds -> leads + a JSON report
13//
14// Three rules this module will not bend:
15//
16//  1. robots.txt is fetched and obeyed before the first request to a host, and
17//     there is no configuration switch that turns it off. If robots.txt is
18//     unreachable the host is treated as fully disallowed (RFC 9309 §2.3.1.4),
19//     because "the rules server is down" is not permission.
20//  2. One host is hit no more often than `delay_ms`, and a `Crawl-delay:` in
21//     robots.txt raises that if it is larger.
22//  3. Nothing is printed or stored that was not extracted from a real response.
23//     A page that could not be fetched produces no leads, not empty ones.
24//
25// Tuning lives in config/agent.json:
26//
27//   "module": {
28//     "user_agent": "AcmeLeadBot/1.0 (+https://acme.example/bot)",
29//     "delay_ms": 750,
30//     "max_depth": 2,
31//     "max_pages": 25,
32//     "same_host_only": true,
33//     "campaigns": {
34//       "q3": { "seeds": ["https://example.com/"], "max_depth": 2 }
35//     }
36//   }
37 
38 
39#include <algorithm>
40#include <cctype>
41#include <cstdlib>
42#include <cstring>
43#include <deque>
44#include <filesystem>
45#include <fstream>
46#include <map>
47#include <mutex>
48#include <set>
49#include <string>
50#include <thread>
51#include <unordered_map>
52#include <unordered_set>
53#include <vector>
54 
55#include "bs9/http.hpp"
56#include "bs9/log.hpp"
57#include "bs9/metrics.hpp"
58#include "bs9/scheduler.hpp"
59#include "leadgen.hpp"
60 
61namespace fs = std::filesystem;
62 
63namespace bs9::leadgen {
64namespace {
65 
66// --------------------------------------------------------------- text bits
67 
68char lower_ascii(char c) {
69  return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
70}
71 
72std::string lower(std::string text) {
73  for (char& c : text) c = lower_ascii(c);
74  return text;
75}
76 
77bool is_space(unsigned char c) {
78  return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
79}
80 
81std::string trim(const std::string& text) {
82  std::size_t begin = 0;
83  std::size_t end = text.size();
84  while (begin < end && is_space(static_cast<unsigned char>(text[begin]))) ++begin;
85  while (end > begin && is_space(static_cast<unsigned char>(text[end - 1]))) --end;
86  return text.substr(begin, end - begin);
87}
88 
89/// Collapses every run of whitespace — including a decoded U+00A0, which is
90/// what &nbsp; becomes and which otherwise glues words together forever.
91bool contains(const std::string& haystack, const std::string& needle) {
92  return haystack.find(needle) != std::string::npos;
93}
94 
95// --------------------------------------------------------------- rate limit
96 
97std::mutex& pace_mutex() {
98  static std::mutex mutex;
99  return mutex;
100}