add some latex snippets because reconstructing them is a pain

Pipfile.lock

@@ -18,19 +18,19 @@
     "default": {
         "beautifulsoup4": {
             "hashes": [
-                "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76",
-                "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11",
-                "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89"
+                "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858",
+                "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348",
+                "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718"
             ],
             "index": "pypi",
-            "version": "==4.6.0"
+            "version": "==4.7.1"
         },
         "certifi": {
             "hashes": [
-                "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296",
-                "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"
+                "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
+                "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
             ],
-            "version": "==2018.1.18"
+            "version": "==2019.3.9"
         },
         "chardet": {
             "hashes": [
@@ -41,59 +41,64 @@
         },
         "idna": {
             "hashes": [
-                "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f",
-                "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4"
+                "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
+                "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
             ],
-            "version": "==2.6"
+            "version": "==2.8"
         },
         "lxml": {
             "hashes": [
-                "sha256:01c45df6d90497c20aa2a07789a41941f9a1029faa30bf725fc7f6d515b1afe9",
-                "sha256:0c9fef4f8d444e337df96c54544aeb85b7215b2ed7483bb6c35de97ac99f1bcd",
-                "sha256:0e3cd94c95d30ba9ca3cff40e9b2a14e1a10a4fd8131105b86c6b61648f57e4b",
-                "sha256:0e7996e9b46b4d8b4ac1c329a00e2d10edcd8380b95d2a676fccabf4c1dd0512",
-                "sha256:1858b1933d483ec5727549d3fe166eeb54229fbd6a9d3d7ea26d2c8a28048058",
-                "sha256:1b164bba1320b14905dcff77da10d5ce9c411ac4acc4fb4ed9a2a4d10fae38c9",
-                "sha256:1b46f37927fa6cd1f3fe34b54f1a23bd5bea1d905657289e08e1297069a1a597",
-                "sha256:231047b05907315ae9a9b6925751f9fd2c479cf7b100fff62485a25e382ca0d4",
-                "sha256:28f0c6652c1b130f1e576b60532f84b19379485eb8da6185c29bd8c9c9bc97bf",
-                "sha256:34d49d0f72dd82b9530322c48b70ac78cca0911275da741c3b1d2f3603c5f295",
-                "sha256:3682a17fbf72d56d7e46db2e80ca23850b79c28cfe75dcd9b82f58808f730909",
-                "sha256:3cf2830b9a6ad7f6e965fa53a768d4d2372a7856f20ffa6ce43d2fe9c0d34b19",
-                "sha256:5b653c9379ce29ce271fbe1010c5396670f018e78b643e21beefbb3dc6d291de",
-                "sha256:65a272821d5d8194358d6b46f3ca727fa56a6b63981606eac737c86d27309cdd",
-                "sha256:691f2cd97cf026c611df1ea5055755eec7f878f2d4f4330dc8686583de6fc5fd",
-                "sha256:6b6379495d3baacf7ed755ac68547c8dff6ce5d37bf370f0b7678888dc1283f9",
-                "sha256:75322a531504d4f383264391d89993a42e286da8821ddc5ac315e57305cb84f0",
-                "sha256:7f457cbda964257f443bac861d3a36732dcba8183149e7818ee2fb7c86901b94",
-                "sha256:7ff1fc76d8804e0f870c343a72007ff587090c218b0f92d8ee784ac2b6eaf5b9",
-                "sha256:8523fbde9c2216f3f2b950cb01ebe52e785eaa8a07ffeb456dd3576ca1b4fb9b",
-                "sha256:8f37627f16e026523fca326f1b5c9a43534862fede6c3e99c2ba6a776d75c1ab",
-                "sha256:a7182ea298cc3555ea56ffbb0748fe0d5e0d81451e2bc16d7f4645cd01b1ca70",
-                "sha256:abbd2fb4a5a04c11b5e04eb146659a0cf67bb237dd3d7ca3b9994d3a9f826e55",
-                "sha256:accc9f6b77bed0a6f267b4fae120f6008a951193d548cdbe9b61fc98a08b1cf8",
-                "sha256:bd88c8ce0d1504fdfd96a35911dd4f3edfb2e560d7cfdb5a3d09aa571ae5fbae",
-                "sha256:c557ad647facb3c0027a9d0af58853f905e85a0a2f04dcb73f8e665272fcdc3a",
-                "sha256:defabb7fbb99f9f7b3e0b24b286a46855caef4776495211b066e9e6592d12b04",
-                "sha256:e2629cdbcad82b83922a3488937632a4983ecc0fed3e5cfbf430d069382eeb9b"
+                "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5",
+                "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f",
+                "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b",
+                "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616",
+                "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b",
+                "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294",
+                "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3",
+                "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90",
+                "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e",
+                "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314",
+                "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f",
+                "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d",
+                "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8",
+                "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3",
+                "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33",
+                "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317",
+                "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63",
+                "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd",
+                "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f",
+                "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a",
+                "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f",
+                "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22",
+                "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d",
+                "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b",
+                "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f",
+                "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86"
             ],
             "index": "pypi",
-            "version": "==4.2.1"
+            "version": "==4.3.3"
         },
         "requests": {
             "hashes": [
-                "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
-                "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
+                "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
+                "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
             ],
             "index": "pypi",
-            "version": "==2.18.4"
+            "version": "==2.21.0"
+        },
+        "soupsieve": {
+            "hashes": [
+                "sha256:6898e82ecb03772a0d82bd0d0a10c0d6dcc342f77e0701d0ec4a8271be465ece",
+                "sha256:b20eff5e564529711544066d7dc0f7661df41232ae263619dede5059799cdfca"
+            ],
+            "version": "==1.9.1"
         },
         "urllib3": {
             "hashes": [
-                "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
-                "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
+                "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
+                "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
             ],
-            "version": "==1.22"
+            "version": "==1.24.2"
         }
     },
     "develop": {}

assets/end.snip

@@ -0,0 +1,7 @@
+
+\end{footnotesize}
+
+\end{multicols}
+
+\end{document}
+

assets/mid1.snip

@@ -0,0 +1 @@
+\section{Single Feats}

assets/mid2.snip

@@ -0,0 +1 @@
+\section{Chained Feats}

assets/start.snip

@@ -0,0 +1,17 @@
+\documentclass[10pt,letterpaper,twoside]{article}
+
+\usepackage{multicol}
+\usepackage{geometry}
+
+\geometry{outer=0.75in,inner=1in,top=0.75in,bottom=0.75in}
+
+\begin{document} 
+
+\begin{multicols}{2}
+
+{\noindent{\LARGE \textbf{Feat Spellbook}}\ \ \ \emph{Level 14}\vspace{0.3em}}
+
+\begin{footnotesize}
+
+\section{Feats Known}
+

queryfeats.py

@@ -8,19 +8,20 @@ from copy import deepcopy
 
 
 charsheet = {
-    'bab': 9,
-    'str': 20,
-    'dex': 14,
-    'con': 14,
-    'int': 16,
+    'bab': 14,
+    'str': 23,
+    'dex': 18,
+    'con': 16,
+    'int': 18,
     'wis': 10,
     'cha': 10,
     'level': [
-        ('fighter', 10)
+        ('fighter', 14)
     ],
     'feat': [
         'toughness',
         'armor proficiency, medium',
+        'armor proficiency, light',
         'power attack',
         'combat stamina',
         'improved unarmed strike',
@@ -53,10 +54,20 @@ charsheet = {
         'martial focus',
         'weapon training class feature',
         'combat reflexes',
+        'dodge',
+        'artful dodge',
+        'cut from the air',
+        'spellcut',
+        'greater weapon focus',
+        'greater weapon focus with selected weapon',
+        'greater weapon specialization',
+        'greater weapon specialization with selected weapon',
         'you have no levels in a class that has the grit class feature',
         'any combat feat',
         'any good alignment',
         'bravery +1 class feature',
+        'bravery +2 class feature',
+        'bravery +3 class feature',
         'human',
         'medium size',
         'no levels in a class that has the favored enemy class feature',
@@ -64,39 +75,45 @@ charsheet = {
         'nonlawful',
         'proficiency with selected weapon',
         'proficient with all martial weapons',
+        'martial weapon proficiency',
         'proficiency with weapon',
         'proficiency with chosen weapon',
         'proficiency with armor spikes',
         'proficient with armor spikes',
         'proficient with scimitar',
         'proficient with sing',
+        'proficient with sling',
         'susceptibility to bleed damage',
         'and proficiency with the selected weapon.',
         'bravery class feature',
+        'proficiency with medium',
     ],
     'skill': [
-        ('knowledge (arcane)', 9),
-        ('knowledge (dungeoneering)', 9),
-        ('knowledge (local)', 9),
-        ('knowledge (nature)', 9),
-        ('knowledge (planes)', 9),
-        ('knowledge (religion)', 9),
-        ('knowledge (engineering)', 1),
+        ('knowledge (arcane)', 14),
+        ('knowledge (dungeoneering)', 14),
+        ('knowledge (local)', 14),
+        ('knowledge (nature)', 14),
+        ('knowledge (planes)', 14),
+        ('knowledge (religion)', 14),
+        ('knowledge (engineering)', 2),
         ('knowledge (geography)', 2),
-        ('knowledge (history)', 4),
-        ('acrobatics', 9),
-        ('climb', 3),
-        ('atealth', 1),
+        ('knowledge (history)', 14),
+        ('acrobatics', 14),
+        ('climb', 4),
+        ('swim', 4),
+        ('stealth', 4),
         ('disable device', 1),
-        ('spellcraft', 1),
+        ('spellcraft', 8),
         ('intimidate', 1),
         ('use magic device', 1),
         ('diplomacy', 1),
         ('perception', 1),
+        ('survival', 1),
         ('linguistics', 4),
-        ('lore (divine battles)', 4),
+        ('lore (divine battles)', 9),
+        ('lore (mythic founts)', 9),
         ('art (anatomy)', 1),
-        ('perform (oratory)', 2),
+        ('perform (oratory)', 1),
     ],
 }
 
@@ -145,9 +162,46 @@ def fillsreqs(pr, charsheet):
             if f != '' and f not in charsheet['feat']:
                 qual = False
 
+    if 'and' in pr:
+        for andpr in pr['and']:
+            if not fillsreqs(andpr, charsheet):
+                qual = False
+
+    if 'or' in pr:
+        orqual = False
+        for orpr in pr['or']:
+            if anyreqs(orpr, charsheet):
+                orqual = True
+                break
+        if not orqual:
+            qual = False
+
     return qual
 
 
+def anyreqs(pr, charsheet):
+    for attr in ['str', 'dex', 'con', 'int', 'wis', 'cha', 'bab']:
+        if attr in pr:
+            if int(pr[attr][0]) <= charsheet[attr]:
+                return True
+    if 'level' in pr:
+        for l in pr['level']:
+            for ll in charsheet['level']:
+                if l[0] == ll[0] and int(l[1]) <= ll[1]:
+                    return True
+    if 'skill' in pr:
+        for s in pr['skill']:
+            for cs in charsheet['skill']:
+                if s[0] == cs[0] and int(s[1]) <= cs[1]:
+                    return True
+    if 'feat' in pr:
+        for f in pr['feat']:
+            if f != '' and f in charsheet['feat']:
+                return True
+
+    return False
+
+
 def qualfeat(feat, charsheet):
     if feat['prereqs']:
         return fillsreqs(feat['prereqs'], charsheet)
@@ -155,22 +209,33 @@ def qualfeat(feat, charsheet):
         return True
 
 
-
-
 def qualfeats(feats, charsheet):
     return [f for f in feats if qualfeat(f, charsheet) and f['name'].lower() not in charsheet['feat']]
 
 
+def addchain(feat, charsheet, depth=0):
+    newchar = deepcopy(charsheet)
+    newchar['feat'].append(feat['name'].lower())
+
+    newfeats = []
+
+    for f in qualfeats(allfeats, newchar):
+        if 'feat' in f['prereqs']:
+            if f['name'].lower() not in newchar['feat'] and feat['name'].lower() in f['prereqs']['feat']:
+                if depth >= 1:
+                    newfeats.append(f)
+                else:
+                    newfeats.append(addchain(f, newchar, depth + 1))
+
+    if len(newfeats) > 0:
+        feat['chain'] = newfeats
+
+    return feat
+
+
 def addchains(feats, charsheet):
-    featnames = [f['name'] for f in feats]
     for f in feats:
-        newreqs = deepcopy(charsheet)
-        newreqs['feat'].append(f['name'].lower())
-        newcandidates = qualfeats(allfeats, newreqs)
-        newfeats = [ nf for nf in newcandidates if nf['name'] not in featnames ]
-
-        if len(newfeats) > 0:
-            f['chain'] = newfeats
+        f = addchain(f, charsheet)
 
 
 def prefeats(feats):
@@ -204,16 +269,18 @@ def asfeat(feat):
 
     return featsnip
 
+
 def aschainfeat(feat):
     featsnip = '\item \\textbf{%s} %s\n\n' % (feat['name'], feat['benefit'])
     if feat['trick']:
         featsnip += '\\textbf{Combat Trick:} %s\n\n' % feat['trick']
     if feat['special']:
         featsnip += '\\textbf{Special:} %s\n\n' % feat['special']
+    if 'chain' in feat:
+        featsnip += chainfeats(feat['chain'])
     return featsnip
 
 
-
 def chainfeats(feats):
     chainsnip = '\\begin{itemize}\n\n'
     for f in feats:
@@ -251,12 +318,16 @@ if __name__ == '__main__':
     singles = []
     chainers = []
     for f in cantake:
-        print(f['name'])
         if 'chain' in f:
             chainers.append(f)
         else:
             singles.append(f)
 
+    # sort both lists
+
+    singles.sort(key=lambda x: x['name'])
+    chainers.sort(key=lambda x: x['name'])
+
     # LaTeXify both lists
 
     singlesnip = ''

scrapefeats.py

@@ -15,19 +15,6 @@ def scrape_featlist(url):
 
     urls = []
 
-    # for pages with tables
-    for featanchor in featspage.xpath('//table/tbody/tr/td[1]/a'):
-        featurl = featanchor.attrib['href']
-        if (featurl.startswith('http:') and 'feats' in featurl
-            and not featurl.endswith('teamwork')
-            and not featurl.endswith('teamwork/')):
-            urls.append(featurl)
-        elif (not featurl.startswith('http:')
-              and not featurl.endswith('teamwork')
-              and not featurl.endswith('teamwork/')):
-            urls.append(url + '/' + featurl)
-
-
     # for pages with subpages list
     for featanchor in featspage.xpath("//ul[@class='ogn-childpages']/li/a"):
         featurl = featanchor.attrib['href']
@@ -52,6 +39,10 @@ def sanitext(text):
     rettext = re.sub(r'â\x80\x99', "'", rettext)
     rettext = re.sub(r'—', '\textemdash', rettext)
     rettext = re.sub(r'"', "''", rettext)
+    rettext = re.sub(r'’', "'", rettext)
+    rettext = re.sub(r'%', "\%", rettext)
+    rettext = re.sub(r'—', " -- ", rettext)
+    rettext = re.sub(r'–', "-", rettext)
     rettext = re.sub(r'ACG', '', rettext)
     rettext = re.sub(r'APG', '', rettext)
     rettext = re.sub(r'ARG', '', rettext)
@@ -75,6 +66,10 @@ def addreq(reqset, t, req):
     return reqset
 
 
+def chomp(text):
+    return text.split(' ', 1)[1]
+
+
 def parse_prereqs(reqtext, sep=r'[,;]'):
     reqs = {}
     for req in [sanitext(x).lower() for x in re.split(sep, reqtext)]:
@@ -129,24 +124,39 @@ def scrape_feat(url):
             'trick': '',
     }
 
-    feattexts = [x for x in
-                 featpage.xpath('//span[@id="ctl00_MainContent_DataListTypes_ctl00_LabelName"]')[0].itertext()]
 
-    for i in range(0,len(feattexts)-1):
-        t = feattexts[i]
-        if i == 0:
-            if t.find('Teamwork') < 0:
-                feat['name'] = sanitext(re.sub(r'\(.*?\)', '', t))
+    nametext = featpage.xpath('//h1')[0].text_content()
+    if nametext.find('Teamwork') < 0:
+        feat['name'] = sanitext(re.sub(r'\(.*?\)', '', nametext))
+    else:
+        return None
+
+    feattexts = [x for x in
+                 featpage.xpath('//div[@class="article-content"]/p')]
+
+    for i in feattexts:
+        t = i.text_content()
+        if t.startswith('Benefit') and feat['benefit'] == '':
+            feat['benefit'] = sanitext(chomp(t))
+        elif t.startswith('Prerequisite') and feat['benefit'] == '':
+            feat['prereqs'] = parse_prereqs(chomp(t))
+        elif t.startswith('Special') and feat['special'] == '':
+            feat['special'] = sanitext(chomp(t))
+        elif t.startswith('Combat Trick'):
+            feat['trick'] = sanitext(chomp(t))
+        else:
+            if feat['benefit'] != '' and not t.startswith('Normal'):
+                feat['benefit'] += '\n\n'
+                feat['benefit'] += sanitext(t)
             else:
-                break
-        elif t == 'Benefit' and feat['benefit'] == '':
-            feat['benefit'] = sanitext(feattexts[i+1])
-        elif t == 'Prerequisites' and feat['benefit'] == '':
-            feat['prereqs'] = parse_prereqs(feattexts[i+1])
-        elif t == 'Special' and feat['special'] == '':
-            feat['special'] = sanitext(feattexts[i+1])
-        elif t == 'Combat Trick':
-            feat['trick'] = sanitext(feattexts[i+7])
+                print('>>> ' + t)
+
+    extralist = [x for x in
+                 featpage.xpath('//div[@class="article-content"]/ul/li')]
+
+    for extra in extralist:
+        feat['benefit'] += '\n\n'
+        feat['benefit'] += '- ' + sanitext(extra.text_content())
 
     if feat['benefit']:
         return feat
@@ -161,20 +171,19 @@ def scrape_feats(baseurls):
 
     feats = []
     for url in urls:
-        print(url)
-         feat = scrape_feat(url)
-    #     if feat:
-    #         feats.append(feat)
-    #     time.sleep(.3)
+        feat = scrape_feat(url)
+        if feat:
+            feats.append(feat)
+        time.sleep(.3)
 
-    # with open('feats.pickle', 'wb') as f:
-    #     pickle.dump(feats, f)
+    with open('feats.pickle', 'wb') as f:
+        pickle.dump(feats, f)
 
 
 
 
 if __name__ == '__main__':
-    scrape_feats(['http://www.d20pfsrd.com/feats/combat-feats',
+    scrape_feats(['http://www.d20pfsrd.com/feats/combat-feats/all-combat-feats',
                   'http://www.d20pfsrd.com/feats/armor-mastery-feats',
                   'http://www.d20pfsrd.com/feats/weapon-mastery-feats'])