WARNING: API error badtoken: Invalid CSRF token. Sleeping for 9.9 seconds, 2020-01-02 22:59:36 Page [[Allen West (musician)]] saved Sleeping for 4.9 seconds, 2020-01-02 22:59:51 Page [[Andrew Cyrille]] saved Sleeping for 7.0 seconds, 2020-01-02 22:59:59 Page [[Andy Chambers]] saved Sleeping for 8.5 seconds, 2020-01-02 23:00:08 Page [[Andy Hurley]] saved Page [[Brad Childress]] saved Page [[Chris Brokaw]] saved Sleeping for 6.0 seconds, 2020-01-02 23:01:05 Page [[Christie Pearce]] saved Sleeping for 6.7 seconds, 2020-01-02 23:01:14 Page [[Chuck Panozzo]] saved Page [[Dana Gioia]] saved Sleeping for 6.1 seconds, 2020-01-02 23:01:37 Page [[Danny Weis]] saved Sleeping for 7.2 seconds, 2020-01-02 23:01:46 Page [[Dave Bailey (musician)]] saved Sleeping for 5.3 seconds, 2020-01-02 23:01:57 Page [[David Gossett]] saved Page [[Eavan Boland]] saved Page [[Frank Funaro]] saved Page [[Jody Rosen]] saved Page [[Keiwan Ratliff]] saved Page [[Mark Duper]] saved Page [[Mike Mangini]] saved Sleeping for 6.8 seconds, 2020-01-02 23:05:14 Page [[Mike Wengren]] saved Page [[Patty Griffin]] saved Page [[Robert Black (Canadian senator)]] saved Page [[Sabina Cvilak]] saved Page [[Shafal Mosed]] saved Page [[Ted Tally]] saved Page [[Tom Dumont]] saved Sleeping for 3.3 seconds, 2020-01-02 23:07:27 Page [[Tony Yeboah]] saved Page [[Vincent Curatola]] saved Page [[Anika Wells]] saved Page [[Ayyan (model)]] saved Sleeping for 7.2 seconds, 2020-01-02 23:08:37 Page [[Barbara Dane]] saved Page [[Christopher Chaplin]] saved Page [[Dave Jones (politician)]] saved Page [[Jason Jones (musician)]] saved Page [[John Tracy (director)]] saved Page [[Juan Cavallaro]] saved Page [[Kieran Mulroney]] saved Sleeping for 8.3 seconds, 2020-01-02 23:11:50 Page [[Kim Matula]] saved Page [[Matt Winston]] saved Page [[Rob Arnold]] saved Page [[Sarai (rapper)]] saved
import pywikibot
import re
from pywikibot import pagegenerators
site = pywikibot.Site('en', 'wikipedia')
def extractfirst(text):
i=0
result=text
result=re.sub("==(a|[^a])*","",result,re.DOTALL)
result=re.sub("\[\[[Cc]ategory:[^\]]*]]","",result)
result=re.sub("\[\[[Ff]ile:[^\]]*]]","",result)
result=re.sub("\[\[[Ii]mage:[^\]]*]]","",result)
result=re.sub("\<nowiki ?\/\>","",result)
result=re.sub("\n"," ",result)
result=re.sub("'{3,5}[^']*'{3,5}","",result)
result=re.sub("''+","",result)
result=re.sub("\[\[([^\|\]\[]*\|)?([^\|\]\[]*)]]",r"\2",result)
while i < 5:
result=re.sub("{{[^{}]*}}","",result)
result=re.sub("\([^\(\)]*\)","",result)
result=re.sub("<ref[^<>]*>[^<>]*<\/ref>","",result)
result=re.sub("<ref[^<>]*\/>","",result)
result=re.sub("<!--[^<>]*-->","",result)
i+=1
result=re.sub("\n","",result)
result=re.sub(" +"," ",result)
result=re.sub("(^.*?(\w\w)[.!?](?=\s[A-Z]|$)).*",r"\1",result)
result=re.sub("^\s*","",result)
result=re.sub("\s*(?=,|\.)","",result)
result=re.sub("\s*$","",result)
return result
def extractdescription(text):
result=text
nationalityregex = "(Afghan|Albanian|Algerian|Andorran|Angolan|Barbuda|Antiguan|Barbudan|Argentine|Armenian|Australian|Austrian|Azerbaijani|Azeri|Bahamas|Bahamian|Bahraini|Bengali|Barbadian|Belarusian|Belgian|Belizean|Beninese|Beninois|Bhutanese|Bolivian|Bosnian|Herzegovinian|Motswana|Botswanan|Brazilian|Bruneian|Bulgarian|Faso|Burkinabé|Burmese|Burundian|Verde|Cabo|Verdean|Cambodian|Cameroonian|Canadian|African|Chadian|Chilean|Chinese|Colombian|Comoran|Comorian|Congolese|Rican|Ivorian|Croatian|Cuban|Cypriot|Republic|Czech|Danish|Djiboutian|Dominican|Republic|Dominican|Timor|Timorese|Ecuadorian|Egyptian|Salvador|Salvadoran|Guinea|Equatorial|Guinean|Equatoguinean|Eritrean|Estonian|Ethiopian|Fijian|Finnish|French|Gabonese|Gambian|Georgian|German|Ghanaian|Gibraltar|Greek|Hellenic|Grenadian|Guatemalan|Guinean|Bissau|Guinean|Guyanese|Haitian|Honduran|Hungarian|Magyar|Icelandic|Indian|Indonesian|Iranian|Persian|Iraqi|Irish|Israeli|Italian|Coast|Ivorian|Jamaican|Japanese|Jordanian|Kazakhstani|Kazakh|Kenyan|Kiribati|Korea|North|Korean|Korea|South|Korean|Kuwaiti|Kyrgyzstani|Kyrgyz|Kirgiz|Kirghiz|Lao|Laotian|Latvian|Lettish|Lebanese|Basotho|Liberian|Libyan|Liechtensteiner|Lithuanian|Luxembourg|Luxembourgish|Macedonian|Malagasy|Malawian|Malaysian|Maldivian|Malian|Malinese|Maltese|Islands|Marshallese|Martiniquais|Martinican|Mauritanian|Mauritian|Mexican|Micronesian|Moldovan|Monégasque|Monacan|Mongolian|Montenegrin|Moroccan|Mozambican|Namibian|Nauruan|Nepali|Nepalese|Dutch|Netherlandic|Zealand|Zealand|Zelanian|Nicaraguan|Nigerien|Nigerian|Marianan|Norwegian|Omani|Pakistani|Palauan|Palestinian|Panamanian|Guinea|Papua|Guinean|Papuan|Paraguayan|Peruvian|Filipino|Philippine|Polish|Portuguese|Rico|Puerto|Rican|Qatari|Romanian|Russian|Rwandan|Kitts|and|Nevis|Kittitian|Nevisian|Saint|Lucian|Saint|Vincentian|Vincentian|Samoan|Marino|Sammarinese|Tomé|Príncipe|São|Toméan|Arabia|Saudi|Arabian|Senegalese|Serbian|Seychellois|Leone|Sierra|Leonean|Singapore|Singaporean|Slovak|Slovenian|Slovene|Islands|Solomon|Island|Somali|African?|South|African|Sudan|South|Sudanese|Spanish|Lanka|Sri|Lankan|Sudanese|Surinamese|Swazi|Swedish|Swiss|Syrian|Tajikistani|Tanzanian|Thai|Leste|Timorese|Togolese|Tokelauan|Tongan|Tobago|Trinidadian|Tobagonian|Tunisian|Turkish|Turkmen|Tuvaluan|Ugandan|Ukrainian|Arab|Emirates|Emirati|Emirian|Emiri|Kingdom|Great|Britain|Northern|Ireland|UK|British|America|United|States|U.S.|American|Uruguayan|Uzbekistani|Uzbek|Vanuatu|Vanuatuan|Vatican|Venezuelan|Vietnamese|Yemeni|Zambian|Zimbabwean)"
if re.search("\.[^\s]\.",result,re.IGNORECASE) or re.search("(br|chan|chapln|dr|fr|gov|miss|mr|mrs|ms|mme|m|msgr|pres|prof|rep|rev|revs|sen|sr|sra|srta|hon|esq|jr|ret|lt|col|sgt|gen|cpl|capt|bg|adm|cwo|ens|maj|msgt|st)\.",result,re.IGNORECASE):
return False
if re.search("(and|or)$",result):
return False
if re.match('^(("?[A-Z][a-z]*"?|(Jr\.)),? )*?(is|was) (a|an) ',result):
result=re.sub('^(("?[A-Z][a-z]*"?|(Jr\.)),? )*?(is|was) (a|an) (.*)',r"\6",result)
result=re.sub(',? who.*',"",result)
result=re.sub(',? and currently.*',"",result)
result=re.sub(',? currently.*',"",result)
result=re.sub(',? as well.*',"",result)
result=re.sub(',? better known.*',"",result)
result=re.sub(',? best known.*',"",result)
result=re.sub(',? also known.*',"",result)
result=re.sub(',? most known.*',"",result)
result=re.sub(',? mostly known.*',"",result)
result=re.sub(',? generally known.*',"",result)
result=re.sub(',? especially known.*',"",result)
result=re.sub(',? internationally known.*',"",result)
result=re.sub(',? well known.*',"",result)
result=re.sub(',? particularly known.*',"",result)
result=re.sub(',? primarily known.*',"",result)
result=re.sub(',? also known.*',"",result)
result=re.sub(',? known for.*',"",result)
result=re.sub(',? riding for( the)?.*',"",result)
result=re.sub(",? active in.*","",result)
result=re.sub(",? born in.*","",result)
result=re.sub(",? perhaps.*","",result)
result=re.sub(",? mainly.*","",result)
result=re.sub(",? [A-Z][a-z]* (is|are|were|was)(a|[^a])*","",result)
result=re.sub("\.$","",result)
result=re.sub("\bformer\b","",result)
result=re.sub('[,;]? (he|she|they) (is|are|were|was).*',"",result)
result=re.sub('([a-zA-Z])', lambda x: x.groups()[0].upper(), result, 1)
result=re.sub("[\.\,\;]$","",result)
result=re.sub(",? (and|is|that|was|were|are|for|of) ?(and|is|that|was|were|are|for|of)?$","",result)
if re.search(", [A-Za-z]+$",result):
return False
if not re.search(' ',result) or re.search('_',result):
return False
if len(result) <= 40:
if re.match(nationalityregex,result):
return result
return False
savecounter = 0
def category_filter(generator, category):
"""
Filter memebers of the specified category out of the generator.
@param generator: Generator to filter
@type generator: iterator
@param category: Category to filter out
@type category: L{pywikibot.pgae.Category}
"""
for page in generator:
if category not in page.categories():
yield page
def dates(text,se):
if re.search("infobox (office ?holder|governor|senator|mayor|politician|chancellor|(vice)? president|congressman|prime minister|mp|member of parlament)",text,re.IGNORECASE):
return "False"
if re.search("(diplomat|republican|democrat|representative|office ?holder|governor|senator|mayor|politician|chancellor|(vice)? president|congressman|prime minister|member of parlament)",sd,re.IGNORECASE):
return "False"
if re.search("(birth based on age as of date|bbad|birth year from age at date)",text,re.IGNORECASE):
return "False"
if re.search("birth_date\s*=\s*[^<}\n]*?(\d\d\d\d)",text,re.IGNORECASE):
return "(born "+re.search("birth_date\s*=\s*[^}\n]*?(\d\d\d\d)",text,re.IGNORECASE).group(1)+")"
if re.search("born[^\)\.]{1,30}?(\d\d\d\d)",re.sub("==(a|[^a])*","",text,re.DOTALL),re.IGNORECASE):
return "(born "+re.search("born[^\)\.]{1,30}?(\d\d\d\d)",text,re.IGNORECASE).group(1)+")"
living_people_cat = pywikibot.Category(site, 'Living people')
sd_article_cat = pywikibot.Category(site, 'Articles with short description')
gen = pagegenerators.CategorizedPageGenerator(living_people_cat,start='Geva')
gen = category_filter(gen, sd_article_cat)
gen = pagegenerators.PreloadingGenerator(gen)
savecounter = 0
for page in gen:
sd = extractdescription(extractfirst(page.text))
if not sd or 'short description' in page.text: # Is the second condition necessary?
continue
datetxt = dates(page.text,sd)
if datetxt is "False":
continue
if datetxt:
sd = str(sd) + " " + str(datetxt)
print(page.title())
description = "{{short description|" + sd + "}}\n"
page.text = description + page.text
savecounter+=1
print(description)
if pywikibot.Page(site, u"User:PearBOT/Biography short descriptions/stop page").text == "":
page.save('Adding automatically generated short description. For more information see [[Wikipedia:Bots/Requests for approval/PearBOT 5]] Feedback appreciated at [[User talk:Trialpears]]')
pass
else:
break
""" This program is dual licensed under the MIT license (provided below) and CC-BY-SA 3.0.
Copyright 2021 Wikipedia user Trialpears
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import pywikibot
import re
from pywikibot import pagegenerators
site = pywikibot.Site('en', 'wikipedia')
def extractfirst(text, paren=True):
i=0
result=text
result=re.sub("\[\[[Cc]ategory:[^\]]*]]","",result)
result=re.sub("\[\[[Ff]ile:[^\]]*]]","",result)
result=re.sub("\[\[[Ii]mage:[^\]]*]]","",result)
result=re.sub("\<nowiki ?\/\>","",result)
result=re.sub("\n"," ",result)
result=re.sub("==(a|[^a])*","",result,re.DOTALL)
result=re.sub("'{3,5}[^']*'{3,5}","",result)
result=re.sub("''+","",result)
result=re.sub("\[\[([^\|\]\[]*\|)?([^\|\]\[]*)]]",r"\2",result)
while i < 5:
result=re.sub("{{[^{}]*}}","",result)
if paren:
result=re.sub("\([^\(\)]*\)","",result)
result=re.sub("<ref[^<>]*>[^<>]*<\/ref>","",result)
result=re.sub("<ref[^<>]*\/>","",result)
result=re.sub("<!--[^<>]*-->","",result)
i+=1
result=re.sub("\n","",result)
result=re.sub(" *"," ",result)
result=re.sub("U\.S\.","US",result)
result=re.sub("(^.*?[.!?])(?=\s[A-Z]|$).*",r"\1",result)
result=re.sub("^\s*","",result)
result=re.sub("\s*(?=,|\.)","",result)
result=re.sub("\s*$","",result)
if re.match('.*(is|was) (a|an) ',result):
result="is a "+re.search('.*(is|was) (a|an) (.*)',result).group(3)
result=re.sub("^, ","",result)
return result
def extractdescription(text):
result=text
nationalityregex = r"(Sri Lankan|Tibetan|Taiwanese|Hong Kong|Hongkongese|Afghan|Albanian|Algerian|Andorran|Angolan|Barbuda|Antiguan|Barbudan|Argentine|Armenian|Australian|Austrian|Azerbaijani|Azeri|Bahamas|Bahamian|Bahraini|Bengali|Barbadian|Belarusian|Belgian|Belizean|Beninese|Beninois|Bhutanese|Bolivian|Bosnian|Herzegovinian|Motswana|Botswanan|Brazilian|Bruneian|Bulgarian|Faso|Burkinabé|Burmese|Burundian|Verde|Cabo|Verdean|Cambodian|Cameroonian|Canadian|African|Chadian|Chilean|Chinese|Colombian|Comoran|Comorian|Congolese|Rican|Ivorian|Croatian|Cuban|Cypriot|Republic|Czech|Danish|Djiboutian|Dominican|Republic|Dominican|Timor|Timorese|Ecuadorian|Egyptian|Salvador|Salvadoran|Guinea|Equatorial|Guinean|Equatoguinean|Eritrean|Estonian|Ethiopian|Fijian|Finnish|French|Gabonese|Gambian|Georgian|German|Ghanaian|Gibraltar|Greek|Hellenic|Grenadian|Guatemalan|Guinean|Bissau|Guinean|Guyanese|Haitian|Honduran|Hungarian|Magyar|Icelandic|Indian|Indonesian|Iranian|Persian|Iraqi|Irish|Israeli|Italian|Coast|Ivorian|Jamaican|Japanese|Jordanian|Kazakhstani|Kazakh|Kenyan|Kiribati|Korea|North|Korean|Korea|South|Korean|Kuwaiti|Kyrgyzstani|Kyrgyz|Kirgiz|Kirghiz|Lao|Laotian|Latvian|Lettish|Lebanese|Basotho|Liberian|Libyan|Liechtensteiner|Lithuanian|Luxembourg|Luxembourgish|Macedonian|Malagasy|Malawian|Malaysian|Maldivian|Malian|Malinese|Maltese|Islands|Marshallese|Martiniquais|Martinican|Mauritanian|Mauritian|Mexican|Micronesian|Moldovan|Monégasque|Monacan|Mongolian|Montenegrin|Moroccan|Mozambican|Namibian|Nauruan|Nepali|Nepalese|Dutch|Netherlandic|Zealand|Zealand|Zelanian|Nicaraguan|Nigerien|Nigerian|Marianan|Norwegian|Omani|Pakistani|Palauan|Palestinian|Panamanian|Guinea|Papua|Guinean|Papuan|Paraguayan|Peruvian|Filipino|Philippine|Polish|Portuguese|Rico|Puerto|Rican|Qatari|Romanian|Russian|Rwandan|Kitts|Nevis|Kittitian|Nevisian|Saint Lucian|Lucian|Vincentian|Vincentian|Samoan|Marino|Sammarinese|Tomé|Príncipe|São|Toméan|Arabia|Saudi|Arabian|Senegalese|Serbian|Seychellois|Leone|Sierra|Leonean|Singapore|Singaporean|Slovak|Slovenian|Slovene|Islands|Solomon|Island|Somali|African?|South|African|Sudan|South|Sudanese|Spanish|Lanka|Sri|Lankan|Sudanese|Surinamese|Swazi|Swedish|Swiss|Syrian|Tajikistani|Tanzanian|Thai|Leste|Timorese|Togolese|Tokelauan|Tongan|Tobago|Trinidadian|Tobagonian|Tunisian|Turkish|Turkmen|Tuvaluan|Ugandan|Ukrainian|Arab|Emirates|Emirati|Emirian|Emiri|Britain|Irish|Ireland|UK|British|America|United States|US|American|Uruguayan|Uzbekistani|Uzbek|Vanuatu|Vanuatuan|Vatican|Venezuelan|Vietnamese|Yemeni|Zambian|Zimbabwean)"
if re.search('(is|was) (a|an) ',result):
result=re.sub(r'^(("?[A-Z][a-z]*"?|(Jr\.)),? )*?(is|was) (a|an) (.*)',r"\6",result)
result=re.sub(r',?( |^)who( |$).*',"",result)
result=re.sub(r',?( |^)whose( |$).*',"",result)
result=re.sub(r",?( |^)who\'s( |$).*","",result)
result=re.sub(r"(^(retired|former)\b ?|(?<= )(retired|former)\b ?)","",result,flags=re.I)
result=re.sub(r"(^professional\b ?|(?<= )professional\b ?)","",result,flags=re.I)
result=re.sub(",? with( |$).*","",result)
result=re.sub(r',? ?( |^)known for( |$).*',"",result)
result=re.sub(r',?( |^)born( |$).*',"",result)
result=re.sub(r',?( |^)from( |$).*',"",result)
result=re.sub(r",?( |^)in( |$).*","",result)
result=re.sub(r",?( |^)for( |$).*","",result)
result=re.sub(r',? ?( |^)that( |$).*',"",result)
result=re.sub(r",? ?( |^)working( |$).*","",result)
result=re.sub(r',? ?( |^)currently( |$).*',"",result)
result=re.sub(r',? ?( |^)as well( |$).*',"",result)
result=re.sub(r',? ?( |^)best( |$).*',"",result)
result=re.sub(r',? ?( |^)also( |$).*',"",result)
result=re.sub(r',? ?( |^)most( |$).*',"",result)
result=re.sub(r',? ?( |^)mostly( |$).*',"",result)
result=re.sub(r',? ?( |^)generally( |$).*',"",result)
result=re.sub(r',? ?( |^)especially( |$).*',"",result)
result=re.sub(r',? ?( |^)internationally known( |$).*',"",result)
result=re.sub(r',? ?( |^)well known( |$).*',"",result)
result=re.sub(r',? ?( |^)primarily known( |$).*',"",result)
result=re.sub(r',? ?( |^)also known( |$).*',"",result)
result=re.sub(r',? ?( |^)known( |$).*',"",result)
result=re.sub(r',? ?( |^)riding( |$).*',"",result)
result=re.sub(r",? ?( |^)active( |$).*","",result)
result=re.sub(r",? ?( |^)involved( |$).*","",result)
result=re.sub(r",? ?( |^)living( |$).*","",result)
result=re.sub(r",? ?( |^)lived( |$).*","",result)
result=re.sub(r",? ?( |^)lives( |$).*","",result)
result=re.sub(r",? ?( |^)resides( |$).*","",result)
result=re.sub(r",? ?( |^)residing( |$).*","",result)
result=re.sub(r",? ?( |^)perhaps( |$).*","",result)
result=re.sub(r",? ?( |^)affiliated( |$).*","",result)
result=re.sub(r",? ?( |^)affiliate( |$).*","",result)
result=re.sub(r",? ?( |^)mainly( |$).*","",result)
result=re.sub(r",? ?( |^)widely( |$).*","",result)
result=re.sub(r",? ?( |^)famous( |$).*","",result)
result=re.sub(r",? ?( |^)noted( |$).*","",result)
result=re.sub(r",? ?( |^)employed( |$).*","",result)
result=re.sub(r",? ?( |^)employee( |$).*","",result)
result=re.sub(r",? ?( |^)associated( |$).*","",result)
result=re.sub(r",? ?( |^)celebrated( |$).*","",result)
result=re.sub(r",? ?( |^)competitor( |$)(?!he|she|they|both|and|is|that|was|were|are|for|a|an|from|the|which|as).*","",result)
result=re.sub(r",? ?( |^)recognized( |$).*","",result)
result=re.sub(r",? ?( |^)considered( |$).*","",result)
result=re.sub(r",? ?( |^)present( |$).*","",result)
result=re.sub(r",? ?( |^)presently( |$).*","",result)
result=re.sub(r",? ?( |^)convicted( |$).*","",result)
result=re.sub(r",? ?( |^)credited( |$).*","",result)
result=re.sub(r",? ?( |^)notable( |$).*","",result)
result=re.sub(r",? ?( |^)coming( |$).*","",result)
result=re.sub(r",? ?( |^)trained( |$).*","",result)
result=re.sub(r",? ?( |^)training( |$).*","",result)
result=re.sub(r",? ?( |^)schooled( |$).*","",result)
result=re.sub(r",? ?( |^)educated( |$).*","",result)
result=re.sub(r"( |^)prominent(?= )","",result,flags=re.I)
result=re.sub(r",? ?( |^)playing( |$).*","",result)
result=re.sub(r",? ?( |^)serving( |$).*","",result)
result=re.sub(r",? ?( |^)founder( |$).*","",result)
result=re.sub(r",? ?( |^)co-?founder( |$).*","",result)
result=re.sub(r",? ?( |^)founded( |$).*","",result)
result=re.sub(r",? ?( |^)co-?founded( |$).*","",result)
result=re.sub(r",? ?( |^)candidate$","",result)
result=re.sub(r",? ?( |^)(now|currently)( |$).*","",result,flags=re.I)
result=re.sub(r"\.$","",result)
result=re.sub(r"( |^)former( |$)","",result)
result=re.sub(r'[,;]? ?\b(he|she|they) (is|are|were|was)( |$).*',"",result)
result=re.sub(r'([a-zA-Z])', lambda x: x.groups()[0].upper(), result, 1)
result=re.sub(r"[\.\,\;]$","",result)
result=re.sub(r",? ?( |^)at .*","",result)
result=re.sub(r",? ?( |^)representing( |$).*","",result)
result=re.sub(r",? ?( |^)commonly( |$).*","",result)
result=re.sub(r",? ?( |^)based( |$).*","",result)
result=re.sub(r",? ?( |^)originally( |$).*","",result)
result=re.sub(r",? ?( |^)specializing( |$).*","",result)
result=re.sub(r",? ?( |^)specialising( |$).*","",result)
result=re.sub(r",? ?( |^)specialized( |$).*","",result)
result=re.sub(r",? ?( |^)specialised( |$).*","",result)
result=re.sub(r",? ?( |^)specializes( |$).*","",result)
result=re.sub(r",? ?( |^)specialises( |$).*","",result)
result=re.sub(r"^Male ([a-z])","$1",result).upper()+re.sub(r"^Male ([a-z])","",result)
result=re.sub(r"^Female ([a-z])","$1",result).upper()+re.sub(r"^Female ([a-z])","",result)
result=re.sub(r",? of "+nationalityregex+r" (descent|heritage|ethnicity)\b.*","",result,flags=re.I)
result=re.sub(r",? of "+nationalityregex+r" and "+nationalityregex+r" (descent|heritage|ethnicity)\b.*","",result,flags=re.I)
result=re.sub(r",? (he|she|they|both|and|is|that|was|were|are|for|a|an|from|the|which|as) ?(he|she|they|both|and|is|that|was|were|are|for|a|an|from|the|which|as)? ?(he|she|they|both|and|is|that|was|were|are|for|a|an|from|the|which|as)? ?(he|she|they|both|and|is|that|was|were|are|for|a|an|from|the|which|as)?$","",result)
result=re.sub(r", and"," and",result)
result=re.sub(r" a player"," player",result)
if not re.search(' ',result) or re.search('_',result):
return False
#if str(re.search(r'^'+nationalityregex+r"$",result)) is "None":
# return False
if len(result) <= 40:
if re.search(nationalityregex,result):
return result
return False
savecounter = 0
def category_filter(generator, category):
"""
Filter memebers of the specified category out of the generator.
@param generator: Generator to filter
@type generator: iterator
@param category: Category to filter out
@type category: L{pywikibot.pgae.Category}
"""
for page in generator:
if category not in page.categories():
yield page
def dates(text,sd):
birthyear = False
deathyear = False
if re.search("infobox (office ?holder|governor|senator|mayor|politician|chancellor|(vice)? president|congressman|prime minister|mp|member of parlament)",text,re.IGNORECASE):
return ""
if re.search("(judge|diplomat|republican|democrat|representative|office ?holder|governor|senator|mayor|politician|chancellor|(vice)? president|congressman|minister|member of parlament)",sd,re.IGNORECASE):
return ""
if re.search("{{(birth based on age as of date|bbad|birth year from age at date)",text,re.IGNORECASE):
return ""
if re.search("birth_date *= *[^}\n=]*?(circa|c\.|\[\[Circa|c\.]]|\[\[circa]])[^}\n]*?(\d\d\d\d)",text,re.IGNORECASE):
birthyear = "c. "+re.search("birth_date *= *[^}\n=]*?(circa|c.|\[\[Circa|c.]]|\[\[circa]])[^}\n]*?(\d\d\d\d)",text,re.IGNORECASE).group(2)
if not birthyear and re.search("(born|\bb\.)[^\)\.]{1,30}?(circa|c\.|\[\[Circa|c\.]]|\[\[circa]])[^\)\.]{0,15}?(\d\d\d\d)",extractfirst(text,False),re.IGNORECASE):
birthyear = "c. "+re.search("(born|\bb\.)[^\)\.]{1,30}?(circa|c\.|\[\[Circa|c.]]|\[\[circa]])[^\)\.]{0,15}?(\d\d\d\d)",extractfirst(text,False),re.IGNORECASE).group(3)
if not birthyear and re.search("birth_date *= *[^<}\n=]*?(\d\d\d\d)",text,re.IGNORECASE):
birthyear = re.search("birth_date *= *[^}\n=]*?(\d\d\d\d)",text,re.IGNORECASE).group(1)
if not birthyear and re.search("(born|\bb\.)[^\)\.]{1,30}?(\d\d\d\d)",extractfirst(text,False),re.IGNORECASE):
birthyear = re.search("(born|\bb\.)[^\)\.]{1,30}?(\d\d\d\d)",extractfirst(text,False),re.IGNORECASE).group(2)
"""
if re.search("death_date\s*=\s*[^}\n]*?(circa|c\.|\[\[Circa|c\.]]|\[\[circa]])[^}\n]*?(\d\d\d\d)",text,re.IGNORECASE):
deathyear = "c. "+re.search("death_date\s*=\s*[^}\n]*?(circa|c.|\[\[Circa|c.]]|\[\[circa]])[^}\n]*?(\d\d\d\d)",text,re.IGNORECASE).group(2)
if not deathyear and re.search("died[^\)\.]{1,30}?(circa|c\.|\[\[Circa|c\.]]|\[\[circa]])[^\)\.]{0,15}?(\d\d\d\d)",re.sub("==(a|[^a])*","",text,re.DOTALL),re.IGNORECASE):
deathyear = "c. "+re.search("died[^\)\.]{1,30}?(circa|c.|\[\[Circa|c.]]|\[\[circa]])[^\)\.]{0,15}?(\d\d\d\d)",text,re.IGNORECASE).group(2)
if not deathyear and re.search("death_date *= *[^<}\n=]*?(\d\d\d\d)",text,re.IGNORECASE):
deathyear = re.search("death_date *= *[^}\n=]*?(\d\d\d\d)",text,re.IGNORECASE).group(1)
if not deathyear and re.search("died[^\)\.]{1,30}?(\d\d\d\d)",re.sub("==(a|[^a])*","",text,re.DOTALL),re.IGNORECASE):
deathyear = re.search("died[^\)\.]{1,30}?(\d\d\d\d)",text,re.IGNORECASE).group(1)
"""
#if deathyear and birthyear:
#return " ("+birthyear+"–"+deathyear+")"
if birthyear:
return " (born "+birthyear+")"
#if deathyear:
#return " (born "+deathyear+")"
#rangeregex = re.search("\([^)]*(\d\d\d\d)[^)-–—―‐‒−-]*[-–—―‐‒−-][^)]*(\d\d\d\d)[^)]*\)",extractfirst(text,False),re.IGNORECASE)
#if rangeregex:
#return " ("+rangeregex.group(1)+"–"+rangeregex.group(2)+")"
return ""
living_people_cat = pywikibot.Category(site, 'Living people')
sd_article_cat = pywikibot.Category(site, 'Articles with short description')
gen = pagegenerators.CategorizedPageGenerator(living_people_cat,start='Arnold')
gen = category_filter(gen, sd_article_cat)
gen = pagegenerators.PreloadingGenerator(gen)
savecounter = 0
for page in gen:
sd = extractdescription(extractfirst(page.text))
if not sd or 'short description' in page.text: # Is the second condition necessary?
continue
datetxt = dates(page.text,sd)
sd = str(sd) + str(datetxt)
description = "{{short description|" + sd + "}}\n"
page.text = description + page.text
#savecounter+=1
if pywikibot.Page(site, u"User:PearBOT/Biography short descriptions/stop page").text == "":
print(page.title())
print(extractfirst(page.text))
print(sd)
page.save('Adding automatically generated short description ([[WP:Bots/Requests for approval/PearBOT 5|BRFA]])')
pass
else:
break
Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en. Retrieving 50 pages from wikipedia:en.
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-35-5276a257b023> in <module> 211 212 savecounter = 0 --> 213 for page in gen: 214 sd = extractdescription(extractfirst(page.text)) 215 if not sd or 'short description' in page.text: # Is the second condition necessary? /srv/paws/pwb/pywikibot/pagegenerators.py in PreloadingGenerator(generator, groupsize) 2142 sites = {} 2143 # build a list of pages for each site found in the iterator -> 2144 for page in generator: 2145 site = page.site 2146 sites.setdefault(site, []).append(page) <ipython-input-35-5276a257b023> in category_filter(generator, category) 161 """ 162 for page in generator: --> 163 if category not in page.categories(): 164 yield page 165 /srv/paws/pwb/pywikibot/data/api.py in __iter__(self) 2633 prev_limit, new_limit, previous_result_had_data) 2634 if not hasattr(self, 'data'): -> 2635 self.data = self.request.submit() 2636 if not self.data or not isinstance(self.data, dict): 2637 pywikibot.debug( /srv/paws/pwb/pywikibot/data/api.py in submit(self) 1810 paramstring) 1811 response, use_get = self._http_request(use_get, uri, body, headers, -> 1812 paramstring) 1813 if response is None: 1814 continue /srv/paws/pwb/pywikibot/data/api.py in _http_request(self, use_get, uri, data, headers, paramstring) 1540 response = http.request(self.site, uri=uri, 1541 method='GET' if use_get else 'POST', -> 1542 data=data, headers=headers) 1543 except Server504Error: 1544 pywikibot.log('Caught HTTP 504 error; retrying') /srv/paws/pwb/pywikibot/tools/__init__.py in wrapper(*__args, **__kw) 1582 del __kw[old_arg] 1583 -> 1584 return obj(*__args, **__kw) 1585 1586 if not __debug__: /srv/paws/pwb/pywikibot/comms/http.py in request(site, uri, headers, **kwargs) 249 250 baseuri = site.base_url(uri) --> 251 r = fetch(baseuri, headers=headers, **kwargs) 252 site.throttle.retry_after = int(r.headers.get('retry-after', 0)) 253 return r /srv/paws/pwb/pywikibot/tools/__init__.py in wrapper(*__args, **__kw) 1582 del __kw[old_arg] 1583 -> 1584 return obj(*__args, **__kw) 1585 1586 if not __debug__: /srv/paws/pwb/pywikibot/comms/http.py in fetch(uri, method, headers, default_error_handling, use_fake_user_agent, **kwargs) 405 response = session.request(method, uri, 406 headers=headers, auth=auth, timeout=timeout, --> 407 **kwargs) 408 except Exception as e: 409 response = e /srv/paws/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json) 528 } 529 send_kwargs.update(settings) --> 530 resp = self.send(prep, **send_kwargs) 531 532 return resp /srv/paws/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs) 641 642 # Send the request --> 643 r = adapter.send(request, **kwargs) 644 645 # Total elapsed time of the request (approximately) /srv/paws/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies) 447 decode_content=False, 448 retries=self.max_retries, --> 449 timeout=timeout 450 ) 451 /srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw) 675 body=body, 676 headers=headers, --> 677 chunked=chunked, 678 ) 679 /srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw) 424 # Python 3 (including for exceptions like SystemExit). 425 # Otherwise it looks like a bug in the code. --> 426 six.raise_from(e, None) 427 except (SocketTimeout, BaseSSLError, SocketError) as e: 428 self._raise_timeout(err=e, url=url, timeout_value=read_timeout) /srv/paws/lib/python3.6/site-packages/urllib3/packages/six.py in raise_from(value, from_value) /srv/paws/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw) 419 # Python 3 420 try: --> 421 httplib_response = conn.getresponse() 422 except BaseException as e: 423 # Remove the TypeError from the exception chain in /usr/lib/python3.6/http/client.py in getresponse(self) 1354 try: 1355 try: -> 1356 response.begin() 1357 except ConnectionError: 1358 self.close() /usr/lib/python3.6/http/client.py in begin(self) 305 # read until we get a non-100 response 306 while True: --> 307 version, status, reason = self._read_status() 308 if status != CONTINUE: 309 break /usr/lib/python3.6/http/client.py in _read_status(self) 266 267 def _read_status(self): --> 268 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") 269 if len(line) > _MAXLINE: 270 raise LineTooLong("status line") /usr/lib/python3.6/socket.py in readinto(self, b) 584 while True: 585 try: --> 586 return self._sock.recv_into(b) 587 except timeout: 588 self._timeout_occurred = True /usr/lib/python3.6/ssl.py in recv_into(self, buffer, nbytes, flags) 1010 "non-zero flags not allowed in calls to recv_into() on %s" % 1011 self.__class__) -> 1012 return self.read(nbytes, buffer) 1013 else: 1014 return socket.recv_into(self, buffer, nbytes, flags) /usr/lib/python3.6/ssl.py in read(self, len, buffer) 872 raise ValueError("Read on closed or unwrapped SSL socket.") 873 try: --> 874 return self._sslobj.read(len, buffer) 875 except SSLError as x: 876 if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: /usr/lib/python3.6/ssl.py in read(self, len, buffer) 629 """ 630 if buffer is not None: --> 631 v = self._sslobj.read(len, buffer) 632 else: 633 v = self._sslobj.read(len) KeyboardInterrupt:
nationalityregex = r"(Sri Lankan|Tibetan|Taiwanese|Hong Kong|Hongkongese|Afghan|Albanian|Algerian|Andorran|Angolan|Barbuda|Antiguan|Barbudan|Argentine|Armenian|Australian|Austrian|Azerbaijani|Azeri|Bahamas|Bahamian|Bahraini|Bengali|Barbadian|Belarusian|Belgian|Belizean|Beninese|Beninois|Bhutanese|Bolivian|Bosnian|Herzegovinian|Motswana|Botswanan|Brazilian|Bruneian|Bulgarian|Faso|Burkinabé|Burmese|Burundian|Verde|Cabo|Verdean|Cambodian|Cameroonian|Canadian|African|Chadian|Chilean|Chinese|Colombian|Comoran|Comorian|Congolese|Rican|Ivorian|Croatian|Cuban|Cypriot|Republic|Czech|Danish|Djiboutian|Dominican|Republic|Dominican|Timor|Timorese|Ecuadorian|Egyptian|Salvador|Salvadoran|Guinea|Equatorial|Guinean|Equatoguinean|Eritrean|Estonian|Ethiopian|Fijian|Finnish|French|Gabonese|Gambian|Georgian|German|Ghanaian|Gibraltar|Greek|Hellenic|Grenadian|Guatemalan|Guinean|Bissau|Guinean|Guyanese|Haitian|Honduran|Hungarian|Magyar|Icelandic|Indian|Indonesian|Iranian|Persian|Iraqi|Irish|Israeli|Italian|Coast|Ivorian|Jamaican|Japanese|Jordanian|Kazakhstani|Kazakh|Kenyan|Kiribati|Korea|North|Korean|Korea|South|Korean|Kuwaiti|Kyrgyzstani|Kyrgyz|Kirgiz|Kirghiz|Lao|Laotian|Latvian|Lettish|Lebanese|Basotho|Liberian|Libyan|Liechtensteiner|Lithuanian|Luxembourg|Luxembourgish|Macedonian|Malagasy|Malawian|Malaysian|Maldivian|Malian|Malinese|Maltese|Islands|Marshallese|Martiniquais|Martinican|Mauritanian|Mauritian|Mexican|Micronesian|Moldovan|Monégasque|Monacan|Mongolian|Montenegrin|Moroccan|Mozambican|Namibian|Nauruan|Nepali|Nepalese|Dutch|Netherlandic|Zealand|Zealand|Zelanian|Nicaraguan|Nigerien|Nigerian|Marianan|Norwegian|Omani|Pakistani|Palauan|Palestinian|Panamanian|Guinea|Papua|Guinean|Papuan|Paraguayan|Peruvian|Filipino|Philippine|Polish|Portuguese|Rico|Puerto|Rican|Qatari|Romanian|Russian|Rwandan|Kitts|Nevis|Kittitian|Nevisian|Saint Lucian|Lucian|Vincentian|Vincentian|Samoan|Marino|Sammarinese|Tomé|Príncipe|São|Toméan|Arabia|Saudi|Arabian|Senegalese|Serbian|Seychellois|Leone|Sierra|Leonean|Singapore|Singaporean|Slovak|Slovenian|Slovene|Islands|Solomon|Island|Somali|African?|South|African|Sudan|South|Sudanese|Spanish|Lanka|Sri|Lankan|Sudanese|Surinamese|Swazi|Swedish|Swiss|Syrian|Tajikistani|Tanzanian|Thai|Leste|Timorese|Togolese|Tokelauan|Tongan|Tobago|Trinidadian|Tobagonian|Tunisian|Turkish|Turkmen|Tuvaluan|Ugandan|Ukrainian|Arab|Emirates|Emirati|Emirian|Emiri|Britain|Irish|Ireland|UK|British|America|United States|US|American|Uruguayan|Uzbekistani|Uzbek|Vanuatu|Vanuatuan|Vatican|Venezuelan|Vietnamese|Yemeni|Zambian|Zimbabwean)"
print(re.search(r'^'+nationalityregex+r"$","Potato"))
None