From 02b243cce89027b1a9cb476505e3fbf8eb8262ab Mon Sep 17 00:00:00 2001
From: Michael Vincerra <michael.vincerra@intel.com>
Date: Thu, 21 Feb 2019 18:44:51 -0800
Subject: [PATCH] Improve code in bundle_lister.py

Modifies extractor function to remove duplicate includes.
- Revises pattern5 regex to include descriptions with special chars.

Signed-off-by: Michael Vincerra <michael.vincerra@intel.com>

Resolves consistency in rows for template.

Signed-off-by: Michael Vincerra <michael.vincerra@intel.com>

Closes #378.

Signed-off-by: Michael Vincerra <michael.vincerra@intel.com>
---
 source/scripts/_python/bundle_lister.py | 26 ++++++++++++-------------
 source/scripts/_python/template.html    |  2 +-
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/source/scripts/_python/bundle_lister.py b/source/scripts/_python/bundle_lister.py
index 8440ab61..a9360e19 100644
--- a/source/scripts/_python/bundle_lister.py
+++ b/source/scripts/_python/bundle_lister.py
@@ -6,7 +6,7 @@ import jinja2
 from jinja2 import Environment, FileSystemLoader, Template
 import git
 from operator import itemgetter
-from datetime import datetime 
+from datetime import datetime
 
 GITHUB_BASE = "https://github.com/clearlinux/clr-bundles/tree/master/bundles/"
 PUNDLES = "https://github.com/clearlinux/clr-bundles/blob/master/packages"
@@ -15,16 +15,14 @@ PATTERN1 = re.compile(r"#\s?\[TITLE]:\w?(.*)")
 PATTERN2 = re.compile(r"#\s?\[DESCRIPTION]:\w?(.*)")
 PATTERN3 = re.compile(r"\(([^()]*|include)\)", re.MULTILINE)
 PATTERN4 = re.compile(r"^((?:(?!#)\w+[^-\s][-])\w+|\w+[^\s-])", re.MULTILINE)
-# ALT PATTERN4 = re.compile(r"^((?:(?!#)(\w+[^-\s])[-]\w+.)[^\s]{1,}[^\s]|\w+[^\s-])", re.MULTILINE)
-
-PATTERN5 = re.compile(r"^(?!=a)\w.+\s[#]\s(\w+.*)?", re.MULTILINE)
-# Previous version: PATTERN5 = re.compile(r"^[^#].*(?<=\s\-\s)(\w+.*)?", re.MULTILINE)
+PATTERN5 = re.compile(r"^\w.+\s[#]\s(\w?.*)?", re.MULTILINE)
 
 def extractor(lines):
     bundle_title = "title"
     data_desc = "description"
     url = "url"
     include_list = []
+    include_unique = []
 
     for i in lines:
         title = PATTERN1.match(i)
@@ -37,26 +35,26 @@ def extractor(lines):
             data_desc = desc.groups(0)[0].strip()
         if url:
             url = os.path.join(GITHUB_BASE, bundle_title)
-
         if includes:
             include_text = includes[0].strip("()")
             include_list.append(include_text)
-    return {"title": bundle_title, "data_desc": data_desc, "include_list": include_list, "url": url}
+            include_unique = set(include_list)
+    return {"title": bundle_title, "data_desc": data_desc, "include_list": include_unique, "url": url}
 
 def pundler():
     with io.open("./cloned_repo/clr-bundles/packages") as file_obj:
         lines = file_obj.readlines()
         pundle_title = "pundle_title"
         pundle_desc = "pundle_desc"
-        purl = "purl" 
+        purl = "purl"
         pundle_list = []
         pun_desc = []
         pundle_master = []
-        
+
         for i in lines:
             pundle = PATTERN4.findall(i)
             pundle_plus = PATTERN5.findall(i)
-            
+
             if pundle:
                 pundle_title = pundle[0]
                 pundle_list.append(pundle_title)
@@ -65,7 +63,7 @@ def pundler():
                 pundle_desc = pundle_plus[0].strip("[]")
                 pun_desc.append(pundle_desc)
 
-        for pun, desc in zip(pundle_list, pun_desc): 
+        for pun, desc in zip(pundle_list, pun_desc):
                 pundle_master.append({"title": pun, "pun_desc": desc, "purl": PUNDLES})
     return pundle_master
 
@@ -82,7 +80,7 @@ def bundler():
                 data.append(extractor(lines))
 
     pundle_master = pundler()
-    data = data + pundle_master 
+    data = data + pundle_master
     filtered = list(filter(lambda x: x.get('title'), data))
     sortedData = sorted(filtered, key=lambda x:x['title'].lower())
     #ALT sortedData2 = sorted(sortedData, key=itemgetter('title'))
@@ -93,6 +91,6 @@ def bundler():
 
     output = template.render(data=sortedData, now=datetime.utcnow())
     with io.open('bundles.html.txt', 'w') as file:
-        file.write(output)   
-          
+        file.write(output)
+
 bundler()
diff --git a/source/scripts/_python/template.html b/source/scripts/_python/template.html
index 416d4e73..7e8d581d 100644
--- a/source/scripts/_python/template.html
+++ b/source/scripts/_python/template.html
@@ -25,8 +25,8 @@
     <tbody>
       <tr>
         <td></td>
+        <td></td>
       </tr>
-      <tr></tr>
       {% for d in data %}
       {% if d.url %}
       <tr id="bundle">