feat(package): Add model packages

Model-packages/linearsvc/LICENSE

@@ -0,0 +1,10 @@
+MIT License
+
+Copyright (c) 2022 Sushant kumar (sushantmishra02102002@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Model-packages/linearsvc/README.md

@@ -0,0 +1,12 @@
+## About
+
+This is a simple python package of Linear support vector machine model trained on the Minerva dataset. The trained model can be used to
+predict the license shortname from the code.
+
+### How to use
+
+- Installing the package:
+  - `pip install linearsvc`
+- Import the trained model:
+  - `from linearsvc import linearsvc`
+  - `short_name = logreg.predict(preprocessed_data)`

Model-packages/linearsvc/setup.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+from os import path
+from io import open
+from setuptools import setup, find_packages
+
+here = path.abspath(path.dirname(__file__))
+
+# fetch the long description from the README.md
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name="linearsvc",
+    version="0.1.1",
+    author="Sushant Kumar",
+    author_email="sushantmishra02102002@gmail.com",
+    description=(
+        "A svm classifier model for predicting license short_name"),
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    package_dir={"": "src"},
+    packages=find_packages(where="src"),
+    include_package_data=True,
+    package_data={
+        'linearsvc': [
+            'data/linearsvc',
+        ]
+    },
+    python_requires=">=3.5"
+)

Model-packages/linearsvc/src/linearsvc/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+
+from pkg_resources import resource_filename
+import pickle
+
+
+class linearsvc():
+    def __init__(self, preprocessed_file):
+        self.preprocessed_file = preprocessed_file
+
+    def classify(self):
+        data = resource_filename("linearsvc", "data/linearsvc")
+        with open(data, 'rb') as f:
+            Classifier = pickle.load(f)
+        return Classifier
+
+    def predict_shortname(self):
+        predictor = self.classify()
+        return predictor.predict(self.preprocessed_file)

Model-packages/linearsvc/src/linearsvc/data/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

Model-packages/linearsvc/src/model_train.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+import pandas as pd
+import pickle
+import os
+from glob import glob
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.svm import LinearSVC
+
+
+def data():
+    folders = glob("Minerva-Dataset-Generation/Split-DB-Foss-Licenses/*")
+    license_lists = []
+    for folder in folders:
+        if os.path.isdir(folder):
+            list = [os.path.join(folder, fname)
+                    for fname in os.listdir(folder)]
+            license_lists.append(list)
+
+    base_lists = []
+    license_texts = []
+    for license_list in license_lists:
+        for license in license_list:
+            path = os.path.dirname(license)
+            base = os.path.basename(path)
+            base_lists.append(base)
+            file = open(license)
+            file_content = file.read()
+            license_texts.append(file_content)
+
+    df = pd.DataFrame({"short_name": base_lists, "text": license_texts})
+
+    df = df.sample(frac=1).reset_index(drop=True)
+    df.dropna(inplace=True)
+    return df
+
+
+def train():
+    train_data = data()
+
+    X_train = train_data.text
+    Y_train = train_data.short_name
+
+    logreg = Pipeline(
+        [
+            ("vect", CountVectorizer()),
+            ("tfidf", TfidfTransformer()),
+            ("clf", LinearSVC(n_jobs=1, C=1e5)),
+        ]
+    )
+    print("Model training has started!")
+    logreg_model = logreg.fit(X_train, Y_train)
+    print("Training finished!")
+
+    with open("./linearsvc/data/linearsvc", "wb") as f:
+        pickle.dump(logreg_model, f)
+    print("Model saved successfully!")
+
+
+if __name__ == "__main__":
+    train()

Model-packages/logreg/LICENSE

@@ -0,0 +1,10 @@
+MIT License
+
+Copyright (c) 2022 Sushant kumar (sushantmishra02102002@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Model-packages/logreg/README.md

@@ -0,0 +1,12 @@
+## About
+
+This is a simple python package of LogisticRegression model trained on the Minerva dataset. The trained model can be used to
+predict the license shortname from the code.
+
+### How to use
+
+- Installing the package:
+  - `pip install logreg`
+- Import the trained model:
+  - `from logreg import logreg`
+  - `short_name = logreg(preprocessed_data)`

Model-packages/logreg/setup.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+from os import path
+from io import open
+from setuptools import setup, find_packages
+
+here = path.abspath(path.dirname(__file__))
+
+# fetch the long description from the README.md
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name="logreg",
+    version="0.1.0",
+    author="Sushant Kumar",
+    author_email="sushantmishra02102002@gmail.com",
+    description=(
+        "A logisticregression model for predicting license short_name"),
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    package_dir={"": "src"},
+    packages=find_packages(where="src"),
+    include_package_data=True,
+    package_data={
+        'logreg': [
+            'data/logreg',
+        ]
+    },
+    python_requires=">=3.5"
+)

Model-packages/logreg/src/logreg/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+
+from pkg_resources import resource_filename
+import pickle
+
+
+class logreg():
+    def __init__(self, preprocessed_file):
+        self.preprocessed_file = preprocessed_file
+
+    def classify(self):
+        data = resource_filename("logreg", "data/logreg")
+        with open(data, 'rb') as f:
+            Classifier = pickle.load(f)
+        return Classifier
+
+    def predict_shortname(self):
+        predictor = self.classify()
+        return predictor.predict(self.preprocessed_file)

Model-packages/logreg/src/logreg/data/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

Model-packages/logreg/src/model_train.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+
+"""
+ Copyright (C) 2022 Sushant Kumar (sushantmishra02102002@gmail.com)
+ SPDX-License-Identifier: GPL-2.0
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+import pandas as pd
+import pickle
+import os
+from glob import glob
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.linear_model import LogisticRegression
+
+
+def data():
+    folders = glob("Minerva-Dataset-Generation/Split-DB-Foss-Licenses/*")
+    license_lists = []
+    for folder in folders:
+        if os.path.isdir(folder):
+            list = [os.path.join(folder, fname)
+                    for fname in os.listdir(folder)]
+            license_lists.append(list)
+
+    base_lists = []
+    license_texts = []
+    for license_list in license_lists:
+        for license in license_list:
+            path = os.path.dirname(license)
+            base = os.path.basename(path)
+            base_lists.append(base)
+            file = open(license)
+            file_content = file.read()
+            license_texts.append(file_content)
+
+    df = pd.DataFrame({"short_name": base_lists, "text": license_texts})
+
+    df = df.sample(frac=1).reset_index(drop=True)
+    df.dropna(inplace=True)
+    return df
+
+
+def train():
+    train_data = data()
+
+    X_train = train_data.text
+    Y_train = train_data.short_name
+
+    logreg = Pipeline(
+        [
+            ("vect", CountVectorizer()),
+            ("tfidf", TfidfTransformer()),
+            ("clf", LogisticRegression(n_jobs=1, C=1e5)),
+        ]
+    )
+    print("Model training has started!")
+    logreg_model = logreg.fit(X_train, Y_train)
+    print("Training finished!")
+
+    with open("./logreg/data/logreg", "wb") as f:
+        pickle.dump(logreg_model, f)
+    print("Model saved successfully!")
+
+
+if __name__ == "__main__":
+    train()