Class: CrawlConstants

[line 45]
Shared constants and enums used by components that are involved in the crawling process

Tags:

AGENT_LIST =  "bz"

[line 201]

ALLOWED_SITES =  'aa'

[line 149]

archive_base_name =  "Archive"

[line 61]

ARCHIVE_CRAWL =  'ay'

[line 174]

ARC_DATA =  "cd"

[line 205]

ARC_DIR =  "cb"

[line 203]

ARC_TYPE =  "cc"

[line 204]

AVERAGE_DESCRIPTION_LENGTH =  'B'

[line 122]

AVERAGE_TITLE_LENGTH =  'A'

[line 121]

AVERAGE_TOTAL_LINK_TEXT_LENGTH =  'C'

[line 123]

BOOST =  'av'

[line 170]

BOTH =  0

[line 50]

Used to say what kind of queue_server this is

BREADTH_FIRST =  'ac'

[line 151]

CACHE_PAGES =  'cn'

[line 215]

CACHE_PAGE_PARTITION =  'ap'

[line 165]

CONTINUE_STATE =  1

[line 86]

CRAWL_DELAY =  'v'

[line 115]

CRAWL_INDEX =  'ba'

[line 176]

CRAWL_ORDER =  'Y'

[line 147]

CRAWL_TIME =  'b'

[line 92]

CRAWL_TYPE =  'az'

[line 175]

CURRENT_SERVER =  "bv"

[line 197]

DATA =  'bt'

[line 195]

data_base_name =  "At"

[line 76]

DESCRIPTION =  't'

[line 113]

DESCRIPTION_LENGTH =  'E'

[line 125]

DESCRIPTION_WORDS =  'H'

[line 128]

DESCRIPTION_WORD_SCORE =  'K'

[line 131]

DISALLOWED_SITES =  'ab'

[line 150]

DNS_TIME =  "by"

[line 200]

DOC_DEPTH =  'M'

[line 133]

DOC_ID = 'am'

[line 162]

DOC_INFO =  'r'

[line 111]

DOC_LEN =  'bi'

[line 184]

DOC_RANK =  'N'

[line 134]

DOMAIN_WEIGHTS =  'bm'

[line 188]

DUMMY =  'V'

[line 143]

ENCODING =  'f'

[line 97]

END_ITERATOR =  'ct'

[line 221]

fetch_archive_iterator =  "FetchArchiveIterator"

[line 63]

fetch_batch_name =  "FetchBatch"

[line 73]

fetch_closed_name =  "FetchClosed"

[line 75]

fetch_crawl_info =  "FetchInfo"

[line 74]

FILETYPE =  'ag'

[line 156]

GENERATION =  'aq'

[line 166]

GOT_ROBOT_TXT =  'p'

[line 109]

HASH =  'o'

[line 108]

HASH_SEEN_URLS = 'aj'

[line 159]

HASH_SUM_SCORE =  'ar'

[line 167]

HASH_URL =  'T'

[line 141]

HASH_URL_COUNT =  'as'

[line 168]

HEADER =  'bb'

[line 177]

HTTP_CODE =  'c'

[line 94]

INDEX =  'z'

[line 119]

INDEXED_FILE_TYPES =  'bq'

[line 192]

INDEXER =  "Indexer"

[line 54]

Used to say what kind of queue_server this is

INDEXING_PLUGINS =  'bl'

[line 187]

index_closed_name =  "IndexClosed"

[line 72]

index_data_base_name =  "IndexData"

[line 68]

INI =  'cq'

[line 218]

INLINKS =  'P'

[line 136]

INVERTED_INDEX =  'i'

[line 101]

IP_ADDRESSES =  'au'

[line 171]

IS_DOC =  'at'

[line 169]

IS_FEED =  'ch'

[line 209]

JUST_METAS =  'aw'

[line 172]

KEY =  "ce"

[line 206]

KEYWORD_LINKS =  'cs'

[line 220]

LANG =  'bg'

[line 182]

LINKS =  'w'

[line 116]

LINK_LENGTH =  'F'

[line 126]

LINK_SEEN_URLS =  'cj'

[line 211]

LINK_WORDS =  'I'

[line 129]

LINK_WORD_SCORE =  'L'

[line 132]

LOCATION =  'bp'

[line 191]

LOGGING =  'cl'

[line 213]

MACHINE =  'h'

[line 100]

MACHINE_ID =  'cf'

[line 207]

MACHINE_URI =  'ae'

[line 154]

MAX =  1

[line 82]

MEMORY_USAGE = 'al'

[line 161]

META_WORDS =  'cm'

[line 214]

MIN =  -1

[line 83]

mirror_table_name =  "mirror_table.txt"

[line 79]

MODIFIED =  'bf'

[line 181]

name_archive_iterator =  "NameArchiveIterator"

[line 62]

NEEDS_OFFSET_FLAG =  0x7FFFFFFF

[line 223]

network_base_name =  "Network"

[line 69]

network_crawllist_base_name =  "NetworkCrawlList"

[line 70]

NEW_CRAWL =  'Q'

[line 138]

NO_DATA_STATE =  2

[line 87]

OFFSET =  'R'

[line 139]

OPERATING_SYSTEM =  'be'

[line 180]

PAGE =  'q'

[line 110]

PAGE_IMPORTANCE =  'ad'

[line 152]

PAGE_RANGE_REQUEST =  'br'

[line 193]

PAGE_RECRAWL_FREQUENCY =  'bs'

[line 194]

PAGE_RULES = 'ao'

[line 164]

PATHS =  'S'

[line 140]

POSITION_LIST =  'bn'

[line 189]

POST_MAX_SIZE =  'ck'

[line 212]

PROXIMITY =  'bo'

[line 190]

queue_base_name =  "QueueBundle"

[line 60]

QUEUE_SERVERS =  "bu"

[line 196]

RECENT_URLS = 'ak'

[line 160]

REDO_STATE =  4

[line 89]

RELEVANCE = 'an'

[line 163]

RESTRICT_SITES_BY_URL =  'Z'

[line 148]

robot_data_base_name =  "RobotData"

[line 67]

ROBOT_INSTANCE =  'bh'

[line 183]

ROBOT_METAS =  "ca"

[line 202]

ROBOT_PATHS =  'n'

[line 107]

robot_table_name =  "robot_table.txt"

[line 78]

ROBOT_TXT =  'x'

[line 117]

SAVED_CRAWL_TIMES =  'j'

[line 103]

save_point =  "SavePoint"

[line 64]

SCHEDULER =  "Scheduler"

[line 58]

Used to say what kind of queue_server this is

schedule_data_base_name =  "ScheduleData"

[line 65]

schedule_name =  "FetchSchedule"

[line 66]

schedule_start_name =  "StartCrawlSchedule.txt"

[line 77]

SCHEDULE_TIME =  'k'

[line 104]

SCORE =  'X'

[line 145]

SEEN_URLS =  'g'

[line 99]

SERVER =  'bc'

[line 178]

SERVER_VERSION =  'bd'

[line 179]

SITES =  'W'

[line 144]

SITE_INFO =  'af'

[line 155]

SIZE =  "bw"

[line 198]

SOURCE_NAME =  'ci'

[line 210]

START_PARTITION =  'cp'

[line 217]

statistics_base_name =  "Statistics"

[line 71]

STATUS =  'a'

[line 91]

STOP_STATE =  -1

[line 85]

SUBDOCS =  'bj'

[line 185]

SUBDOCTYPE =  'bk'

[line 186]

SUMMARY =  'ah'

[line 157]

SUMMARY_OFFSET =  'U'

[line 142]

THUMB =  'u'

[line 114]

TIMESTAMP =  'd'

[line 95]

TITLE =  's'

[line 112]

TITLE_LENGTH =  'D'

[line 124]

TITLE_WORDS =  'G'

[line 127]

TITLE_WORD_SCORE =  'J'

[line 130]

TOTAL_TIME =  "bx"

[line 199]

TO_CRAWL =  'y'

[line 118]

TYPE =  'e'

[line 96]

UI_FLAGS =  'cr'

[line 219]

URL =  'l'

[line 105]

URL_INFO =  'ai'

[line 158]

URL_WEIGHT =  'O'

[line 135]

VIDEO_SOURCES =  'cg'

[line 208]

WAITING_START_MESSAGE_STATE =  3

[line 88]

WARC_ID =  'co'

[line 216]

WEB_CRAWL =  'ax'

[line 173]

WEIGHT =  'm'

[line 106]