Almost done, need just add the parameter matching and writing the sni…

…ppet to the output. beaknit#34

build/build-snippets.py

@@ -2,13 +2,12 @@
 # vim: ai ts=4 sts=4 et sw=4
 """Build snippets from documentation from AWS."""
 import requests
-import sys
 import collections
 from templating import build_with_template
 from lxml import html
 
 
-BASE_HREF = 'http://docs.aws.amazon.com/pt_br/AWSCloudFormation/latest/UserGuide/'
+BASE_HREF = 'http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/'
 
 
 def build_index():
@@ -22,7 +21,9 @@ def build_index():
 
     for e in tree:
             arn = e.text_content()
-            title = e.text_content().replace('::', '-').replace('AWS-', "").lower()
+            title = e.text_content().replace('::', '-') \
+                .replace('AWS-', "") \
+                .lower()
             href = e.get('href')
             full_href = BASE_HREF + href
             index[arn] = (arn, title, href, full_href)
@@ -32,26 +33,44 @@ def build_index():
 
 def generate(index):
     """Will traverse index and generate all the snippets."""
-    default_suffix = '.sublime-snippet'
     for k, v in index.iteritems():
         (arn, title, href, full_href) = v
         snippet = createSnippet(arn, title, href, full_href)
+        writeToOutput(arn, snippet)
 
 
 def createSnippet(arn, title, href, full_href):
     """Create a snippet with the args."""
     elem_expr = '//*[@id="main-col-body"]/div/div/pre/code'
+
+    # uses an old documentation format
+    if (arn == 'AWS::SDB::Domain'):
+        elem_expr = '//*[@id="main-col-body"]/div/pre'
+
     page = requests.get(full_href)
-    doc = doc = html.fromstring(page.text.decode('utf-8'))
+    doc = html.fromstring(page.text)
     tree = doc.xpath(elem_expr)
-    print 'creating {} {} {}'.format(arn, title, tree)
-    return build_with_template(arn, title, tree)
+    print 'creating {} => {}'.format(arn, title)
+    return build_with_template(arn, title, tree, full_href)
+
+
+def safedebug(index, k):
+    """Debug helper for scrapping problems."""
+    (arn, title, href, full_href) = index[k]
+    snippet = createSnippet(arn, title, href, full_href)
+    print snippet
+
+
+def writeToOutput(arn, snippet):
+    """Write the snippet to the output folder."""
+    default_suffix = ".sublime-snippet"
 
 
 def main():
     """Main doc generation."""
     index = build_index()
-    generate(index)
+    # generate(index)
+    safedebug(index, 'AWS::SSM::Document')
 
 if __name__ == "__main__":
     main()

build/templating.py

@@ -2,23 +2,28 @@
 # vim: ai ts=4 sts=4 et sw=4
 """Templating operations for the snippets created by the build script."""
 
-
+from __future__ import unicode_literals
 import sys
 
-
-def build_with_template(trigger, title, body):
+def build_with_template(trigger, title, body, full_href):
     """With all args give, transform body with param list and return a complete snippet."""
 
     processed_body = """"${{1:-}}" : {}""".format((body[0]).text_content())
 
     template = """
+    <!-- Auto-generated from {} -->
     <snippet>
         <content><![CDATA[{}]]></content>
         <!-- Optional: Set a tabTrigger to define how to trigger the snippet -->
         <tabTrigger>{}</tabTrigger>
         <!-- Optional: Set a scope to limit where the snippet will trigger -->
         <scope>source.cloudformation</scope>
     </snippet>
-    """.format(processed_body, title)
+    """.format(full_href, processed_body, title)
+
+    transformed_template = transform(template)
+
+    return transformed_template
 
-    return template
+def transform(template):
+    return template.replace('JSON object', '[]')