ETLConnector4ODK.py

# -*- coding: utf-8 -*-
"""
/***************************************************************************
 ETLConnector4ODK
                                 A QGIS plugin
 Plugin for extraction, transformation and loading data from ODK server to QGIS
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2022-10-03
        git sha              : $Format:%H$
        copyright            : (C) 2022 by Derrick Demeveng
        email                : demeveng@gmail.com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
from qgis.PyQt.QtCore import QSettings, QTranslator, QCoreApplication, QVariant
from qgis.PyQt.QtGui import QIcon
from qgis.PyQt.QtWidgets import QAction, QMessageBox, QFileDialog, QTableWidgetItem
from qgis.core import QgsFields, QgsField, QgsFeature, QgsProject, QgsVectorFileWriter, QgsGeometry, QgsPointXY, QgsWkbTypes, QgsVectorLayer, QgsVectorDataProvider
#from PyQt5.QtCore import QTimer
# Initialize Qt resources from file resources.py
import requests as rq
import csv

# Initialize Qt resources from file resources.py
from .resources import *
# Import the code for the dialog
from .ETLConnector4ODK_dialog import ETLConnector4ODKDialog
import os.path
import webbrowser

class ETLConnector4ODK:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface
        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)
        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir,
            'i18n',
            'ETLConnector4ODK_{}.qm'.format(locale))

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)
            QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&ETL Connector for ODK')

        # Check if plugin was started the first time in current QGIS session
        # Must be set in initGui() to survive plugin reloads
        self.first_start = None

        # project global variables
        self.keysList = list()
        self.dataList = list()

        #self.timer = QTimer()
        #self.timer.timeout.connect(self.on_timer_timeout) #runs the auto update function

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('ETLConnector4ODK', message)


    def add_action(
        self,
        icon_path,
        text,
        callback,
        enabled_flag=True,
        add_to_menu=True,
        add_to_toolbar=True,
        status_tip=None,
        whats_this=None,
        parent=None):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            # Adds plugin icon to Plugins toolbar
            self.iface.addToolBarIcon(action)

        if add_to_menu:
            self.iface.addPluginToMenu(
                self.menu,
                action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/ETLConnector4ODK/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'ETL Connector for ODK'),
            callback=self.run,
            parent=self.iface.mainWindow())

        # will be set False in run()
        self.first_start = True


    def unload(self):
        """Removes the plugin menu item and icon from QGIS GUI."""
        for action in self.actions:
            self.iface.removePluginMenu(
                self.tr(u'&ETL Connector for ODK'),
                action)
            self.iface.removeToolBarIcon(action)


    def run(self):
        """Run method that performs all the real work"""

        # Create the dialog with elements (after translation) and keep reference
        # Only create GUI ONCE in callback, so that it will only load when the plugin is started
        if self.first_start == True:
            self.first_start = False
            self.dlg = ETLConnector4ODKDialog()
            self.dlg.connect.clicked.connect(self.connectionStatus)
            self.dlg.previewData.clicked.connect(self.previewJsonData)
            self.dlg.browsOutput.clicked.connect(self.fileBrowser)
            self.dlg.singleGeo.toggled.connect(self.geometryState)
            self.dlg.noGeometry.toggled.connect(self.noGeometryState)
            #self.dlg.addPcodes.toggled.connect(self.enablePcodes)
            self.dlg.closeMain.clicked.connect(self.closeWindow)
            self.dlg.runProcess.clicked.connect(self.process)
            self.dlg.help.clicked.connect(self.browser)
            #self.dlg.updateLayer.toggled.connect(self.layerUpdateState)
            #self.dlg.updateLayer.stateChanged.connect(self.toggle_auto_update)
            #self.dlg.tabWidget.currentChanged(0).connect(self.changeButtonText)
            #self.dlg.tabWidget.currentChanged(1).connect(self.changeButtonText)
            #QMessageBox.information(self.dlg, "Message", "This is an ODK2QGIS Connector")

        # show the dialog
        self.dlg.show()
        # Run the dialog event loop
        result = self.dlg.exec_()
        # See if OK was pressed
        if result:
            # Do something useful here - delete the line containing pass and
            # substitute with your code.
            pass

#======================FUNCTIONS===========================================
    #Slot functions
    def connectionStatus(self):
        global data
        self.dlg.connectionResult.clear()
        try:
            #url = "https://kc.kobotoolbox.org/api/v1/data/648039?format=json" #geopoint
            #url = "https://kc.kobotoolbox.org/api/v1/data/1764175?format=json" #geotrace
            #url = "https://kc.kobotoolbox.org/api/v1/data/1768435?format=json" #repeat geotrace
            #url = "https://api.whonghub.org/api/v1/data/11139.json" #polygon
            #url = "https://api.whonghub.org/api/v1/data/11138.json" #line
            #url = "https://api.whonghub.org/api/v1/data/9815.json" #repeat line
            #url = "https://api.whonghub.org/api/v1/data/4069.json" #repeat point
            #url = "https://api.whonghub.org/api/v1/data/9506.json" #point
            url = self.dlg.url.text()
            #user=""
            #user =""
            user = self.dlg.userName.text()
            #pwd = ""
            #pwd = ""
            pwd = self.dlg.password.text()
            data = rq.get(url, auth = (user,pwd))
            #print(data)
            if data.status_code == 200:
                self.dlg.connectionResult.setText('<span style=\" color: #006400;\">%s</span>' % ("Connection sucessful with code : "+ str(data.status_code)))
            else:
                self.dlg.connectionResult.setText('<span style=\" color: #ff0000;\">%s</span>' % ("Connection failed with code : "+ str(data.status_code)))
        except Exception as e:
            self.dlg.connectionResult.setText('<span style=\" color: #ff0000;\">%s</span>' % ("Connection faild with error message : "+ str(e)))

    def previewJsonData(self):
        #global keysList #list containing the keys (column names of the data)
        #global dataList #List containing the row data in form of lists
        self.dataList = list() #contains the data in a list
        # clear all widgets
        self.dlg.jsonText.clear()
        self.dlg.dataTable.clear()
        self.dlg.tableSize.clear()
        self.dlg.latitude.clear()
        self.dlg.longitude.clear()
        self.dlg.geometry.clear()
        #self.dlg.adminColumn.clear()
        # function to preview the data in the Qplaintextwidget and the qtablewiget
        jsonData = data.json() #load the data into a json data structure
        self.keysList = list() # get the keys of the data
        # get all possible keys in the json file. the looping ensures that all fields are captured when the length of the dicts are not equal
        for i in jsonData:
            self.dlg.jsonText.appendPlainText("----" + str(list(i.values())) + "\n")
            for j in i.keys():
                if j in self.keysList:
                    continue
                else:
                    self.keysList.append(j)

        #self.dlg.jsonText.appendPlainText(str(self.keysList) + "\n") # display the columns names in the QPlaintextwiget
        #loop through the data and display the rows in the QPlaintextwidget
        for i in jsonData:
            #self.dlg.jsonText.appendPlainText(str(list(i.values())) + "\n") # display the data in the QPlaintextwiget
            li = list() # list to contain row data making sure they have the same number of inputs as the number of comlumns to avoid data mismatch in the output file

            for j in self.keysList:
                if j in i.keys():
                    li.append(i[j])
                else:
                    li.append("")
            self.dataList.append(li)

        #self.dlg.jsonText.appendPlainText(str(self.dataList) + "\n")
        #populate the QTablewiget
        self.columnCount = self.dlg.dataTable.columnCount()
        for i in range(0,self.columnCount):
            self.dlg.dataTable.removeColumn(0)
        
        for i in range(0, len(self.keysList)): # create columns objects in the table before populating them with values
            self.dlg.dataTable.insertColumn(i)

        self.columns = self.dlg.dataTable.setHorizontalHeaderLabels(self.keysList) #display the columns
        self.dlg.dataTable.setRowCount(20) #set the length of the table rows
        row = 0 #row number
        for d in jsonData[0:20]:# loop through the json data and populate the qtablewiget
            column = 0
            rowKeys = d.keys()
            for key in self.keysList:
                if key in rowKeys:
                    self.dlg.dataTable.setItem(row,column, QTableWidgetItem(str(d[key])))
                    column += 1
                else:
                    self.dlg.dataTable.setItem(row,column, QTableWidgetItem(""))
                    column += 1
            row += 1
        #Populate qplaintext and qdropdown wigets with data
        self.dlg.tableSize.setText('<span style=\" color: #006400;\">%s</span>' %("Table size : " + str(len(self.dataList)) + " Rows x " + str(len(self.keysList)) + " Columns"))
        self.dlg.latitude.addItems(self.keysList)
        self.dlg.longitude.addItems(self.keysList)
        self.dlg.geometry.addItems(self.keysList)
        #self.dlg.adminColumn.addItems(self.keysList)

    def fileBrowser(self): #brows and create the output file data in .gpkg and .shp
        fname, format = QFileDialog.getSaveFileName(self.dlg, 'Save file', 'c:\\',"GPKG (*.gpkg);;ESRI Shapefile (*.shp);;comma-seperated values (*.csv)")
        if self.dlg.noGeometry.isChecked():
            fname = os.path.join(os.path.dirname(fname),os.path.basename(fname).split(".")[0] + ".csv")
        self.dlg.outputFile.clear()
        self.dlg.outputFile.setText(fname)
        #self.dlg.jsonText.appendPlainText(format)
    def geometryState(self):
        #toggle the geometry combobox state
        if self.dlg.singleGeo.isChecked():
            self.dlg.geometry.setEnabled(True)
            self.dlg.latitude.setEnabled(False)
            self.dlg.longitude.setEnabled(False)
        else:
            self.dlg.geometry.setEnabled(False)
            self.dlg.latitude.setEnabled(True)
            self.dlg.longitude.setEnabled(True)
    
    def noGeometryState(self):
        #toggle the no-geometry combobox state
        if self.dlg.noGeometry.isChecked():
            self.dlg.singleGeo.setEnabled(False)
            self.dlg.geometry.setEnabled(False)
            self.dlg.latitude.setEnabled(False)
            self.dlg.longitude.setEnabled(False)
            self.dlg.updateLayer.setEnabled(False)
        else:
            self.dlg.updateLayer.setEnabled(True)
            self.dlg.singleGeo.setEnabled(True)
            self.geometryState()
    
    """ def enablePcodes(self):
        #toggle the pcodes combobox state
        if self.dlg.addPcodes.isChecked():
            self.dlg.adminColumn.setEnabled(True)
            self.dlg.adminLevel.setEnabled(True)
        else:
            self.dlg.adminColumn.setEnabled(False)
            self.dlg.adminLevel.setEnabled(False) """
    
    def splitLatLongAltPrec(self):
        geometryColumn = self.dlg.geometry.currentText()

    def createGeoPoint(self, filePath, format, crs):
        pass

    def createGeoTrace(self, filePath, format, crs):
        pass

    def createGeoShape(self, filePath, format, crs):
        pass  

    def checkGeometryType(self):
        pass

    def createFeatureClass(self, filePath, format, crs):
        #create a list to store coordnates of lines
        lines_coordinates = list() # coordinates of the multi-lines
        line_coordinates = list() # coordinates of one line
        points = []
        polygons = []
        geomCol = self.dlg.geometry.currentText()
        #geomCol = '_geolocation'
        index = self.keysList.index(geomCol)
        
        #self.dlg.jsonText.clear()
        #self.dlg.jsonText.appendPlainText("Check 1")
        # define fields for feature attributes. A QgsFields object is needed
        featureFields = QgsFields()
        # populate the fields with the column names stored in Keylist
        for field in self.keysList:
            #self.dlg.jsonText.appendPlainText(field)
            featureFields.append(QgsField(field, QVariant.String)) #.replace("/","_")
        
        crs = crs #QgsProject.instance().crs()
        transform_context = QgsProject.instance().transformContext()
        save_options = QgsVectorFileWriter.SaveVectorOptions()
        if format == "shp":
            save_options.driverName = "ESRI Shapefile" #output format of featureclass ex: "ESRI Shapefile"
        elif format == "gpkg":
            save_options.driverName = "GPKG"

        save_options.fileEncoding = "UTF-8"
        #self.dlg.jsonText.appendPlainText("Check 2")

        # Check the geometry type to be created
        if type(self.dataList[0][index]) == type(str("a")): # This checks for geopoints, geotrace and geoshape
            s = self.dataList[0][index].split(";")
            if len(s) > 1:
                if s[0] == s[-1]: # if the first and the last point of the list are the same, then it's a polygon
                    writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.Polygon, crs, transform_context, save_options)
                    #self.dlg.jsonText.appendPlainText("Check 3")
                else: # else it's a line
                    writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.LineString, crs, transform_context, save_options)
                    #self.dlg.jsonText.appendPlainText("Check 4")
            elif len(s) == 1: # if the length is 1, then it's a point
                writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.Point, crs, transform_context, save_options)
                #self.dlg.jsonText.appendPlainText("Check 5")

        elif type(self.dataList[0][index]) == type(list()): # This checks for repeat geopoints (multi-points), repeat geotrace (multi-line) and repeat geoshape (multi-polygon)
            #s = self.dataList[0][index]
            if type(self.dataList[0][index][0]) == type(dict()):
                if "{}/geopoint".format(geomCol) in self.dataList[0][index][0].keys(): # if the first and the last point of the list are the same, then it's a polygon
                    writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.MultiPoint, crs, transform_context, save_options)
                    #self.dlg.jsonText.appendPlainText("Check 18")
                elif "{}/geotrace".format(geomCol) in self.dataList[0][index][0].keys(): # else it's a line
                    writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.MultiLineString, crs, transform_context, save_options)
                    #self.dlg.jsonText.appendPlainText("Check 17")
                elif "{}/geoshape".format(geomCol) in self.dataList[0][index][0].keys():
                    writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.MultiPolygon, crs, transform_context, save_options)
                    #self.dlg.jsonText.appendPlainText("Check 15")
            else: # if the length is 1, then it's a point
                writer = QgsVectorFileWriter.create(filePath, featureFields, QgsWkbTypes.Point, crs, transform_context, save_options)
                #self.dlg.jsonText.appendPlainText("Check 16")

        if writer.hasError() != QgsVectorFileWriter.NoError:
            self.dlg.processingMsg.setText('<span style=\" color: #ff0000;\">%s</span>' % ("Error when creating file: ",  writer.errorMessage()))

        # create QGIS feature
        fet = QgsFeature()
        #self.dlg.jsonText.appendPlainText("Check 6")

        # Start populating the data
        for row in self.dataList:
            i = 0
            row2 = row.copy()

            lenIndex = ""
            if type(self.dataList[0][index]) != type(list()):
                lenIndex = len(row[index].split(";"))

            #self.dlg.jsonText.appendPlainText(str(row2))
            #if self.dlg.singleGeo.isChecked():# do when a single column geometry is checked
            # Below is the structure we need to fit into a QGIS multi-line feature class
            """ lines_coordinates = [
                [(0, 0), (1, 1)],  # Line 1
                [(1, 1), (2, 3)],  # Line 2
                [(2, 3), (4, 4)]   # Line 3
            ] """

            #self.dlg.jsonText.appendPlainText("Check 7")
            """
                There are two types of scenarios considered here. Either we are dealing with geopoints, geotrace or geoshapes.

                If it is a geopoint, it is represented in two ways when pulled from the API
                1-  [0, 0] : which represent the latitude and longitude in the for of a list
                2- 0 0 0 0 : Whick represent the lat, long, alt, and precision in form of string.

                Geotrace is represented in the form of string coordinates seperated by ;.
                example : '-4.2755481 15.2788814 277.1000061035156 5.35; -4.2757848 15.278532 277.6000061035156 15.287; 
                            -4.2757787 15.2785256 277.0 10.806; -4.2757776 15.2785242 277.29998779296875 10.882; 
                            -4.2751764 15.2781687 285.0 7.065; -4.2756798 15.2768732 283.3999938964844 4.644'.
                
                A geoshape will have the same structure as the geotrace except that the first and the last coordinates are always the same.

                On the other hand, we also have situations where the geopoints, geotrace and geoshapes are repeated using the repeat table functionality
                of ODK producing multi-points, multi-trace and multi-polygons. Records in this case are stored in a list of dictionaries
                where every dictionary in a list represent one feature.

                example repeat geotrace: [{'segment/geotrace': '-4.2755481 15.2788814 277.1000061035156 5.35; -4.2757848 15.278532 277.6000061035156 15.287; -4.2757787 15.2785256 277.0 10.806; -4.2757776 15.2785242 277.29998779296875 10.882; -4.2751764 15.2781687 285.0 7.065; -4.2756798 15.2768732 283.3999938964844 4.644', 'segment/point_pic': '1700649549839.jpg', 'segment/point_position': '1', 'segment/point_details/width': 20, 'segment/point_details/debris': 'moderate', 'segment/point_details/junction': 'sewer', 'segment/point_details/width_unit': 'meters'}, 
                          {'segment/geotrace': '-4.2757748 15.2786011 277.29998779296875 5.035; -4.2758051 15.2786253 277.1000061035156 5.426; -4.2758155 15.2786326 277.1000061035156 7.554; -4.2769991 15.2790314 275.6000061035156 9.18; -4.2777149 15.2785795 276.0 3.974', 'segment/point_pic': '1700650833741.jpg', 'segment/point_position': '2', 'segment/point_details/debris': 'moderate', 'segment/point_details/junction': 'sewer', 'segment/point_details/width_unit': 'meters'}
                        ] 

                example repeat geopoint: [{'segment/geopoint': '-1.9386298 30.0664064 1369.9568239480448 10.0', 'segment/point_pic': '1664793972340.jpg', 'segment/point_position': '1'}, 
                         {'segment/geopoint': '-1.9388676 30.0662867 1374.1872454725058 9.714', 'segment/point_position': '2'}, 
                         {'segment/geopoint': '-1.9390337 30.0660783 1383.3687905605484 9.666', 'segment/point_position': '3'}, 
                         {'segment/geopoint': '-1.9390337 30.0660783 1383.2919118012921 9.428', 'segment/point_position': '4'}]
            """
            if isinstance(row[index], list):
                if isinstance(row[index][0], dict):
                    if '{}/geotrace'.format(geomCol) in row[index][0].keys():

                        #self.dlg.jsonText.appendPlainText("Nothing to save yet")
                        #geomCol = "segment"
                        #index = self.keysList.index(geomCol)
                        #self.dlg.jsonText.appendPlainText(row[index][1])
                        for k in row[index]:
                            for j in k['{}/geotrace'.format(geomCol)].split(";"):
                                j = j.strip() # Remove leading/trailing spaces
                                if j:  # Check if the string is not empty
                                    # Create multi-line geometry from the list of coordinate
                                    coordinates = j.split(" ") #long, lat, alt, precision
                                    line_coordinates.append((coordinates[0], coordinates[1]))
                                    #self.dlg.jsonText.appendPlainText("reached here")
                                    #self.dlg.jsonText.appendPlainText(j.split(" ")[0])
                            lines_coordinates.append(line_coordinates)
                            line_coordinates = list() # reset the list for a new line

                        multi_line = QgsGeometry.fromMultiPolylineXY([[QgsPointXY(float(y), float(x)) for x, y in line] for line in lines_coordinates])  
                        fet.setGeometry(multi_line)
                        lines_coordinates = list() # reset the list for new coordinates
                        #self.dlg.jsonText.appendPlainText("Check 8")
                    
                    elif '{}/geoshape'.format(geomCol) in row[index][0].keys():
                        for k in row[index]:
                            coordinates = k['segment/geoshape'].split(';')
                            points = [QgsPointXY(float(coord.split()[1]), float(coord.split()[0])) for coord in coordinates]
                            polygons.append([points])  # Each polygon is a list of points
                        
                        # Create a multipolygon feature from the polygons
                        multipolygon = QgsGeometry.fromMultiPolygonXY(polygons)
                        fet.setGeometry(multipolygon)
                        polygons = []

                    elif '{}/geopoint'.format(geomCol) in row[index][0].keys():
                        for k in row[index]:
                            lat, lon = map(float, k['segment/geopoint'].split()[:2])
                            points.append(QgsPointXY(lon, lat))

                        # Create a multipoint feature from the points
                        multipoint = QgsGeometry.fromMultiPointXY(points)
                        fet.setGeometry(multipoint)
                        points = []
                else:
                    #self.dlg.jsonText.appendPlainText("Check 13")
                    fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(row[index][1]),float(row[index][0]))))
                    #self.dlg.jsonText.appendPlainText("I was here before.................LOL")
                    #self.dlg.jsonText.appendPlainText(str(float(row[index][1])))
                    

            elif lenIndex > 1: # check if the feature is a point, line or polygone
                split = row[index].split(";")
                #self.dlg.jsonText.appendPlainText("Check 9")
                if split[0] == split[-1]:
                    #self.dlg.jsonText.appendPlainText("Check 10")
                    points = [QgsPointXY(float(c.split()[1]), float(c.split()[0])) for c in split]
                    poly = QgsGeometry.fromPolygonXY([points])
                    fet.setGeometry(poly)
                    #self.dlg.jsonText.appendPlainText("Check 11")
                else:
                    #self.dlg.jsonText.appendPlainText("Check 12")
                    self.dlg.jsonText.appendPlainText(str(split))
                    points = [QgsPointXY(float(c.split()[1]), float(c.split()[0])) for c in split]
                    line = QgsGeometry.fromPolylineXY(points)
                    fet.setGeometry(line)

            elif lenIndex == 1:
                #self.dlg.jsonText.appendPlainText("Check 14")
                #self.dlg.jsonText.appendPlainText(row[index])
                fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(row[index].split(" ")[1]),float(row[index].split(" ")[0]))))

            """ else: # do when lon and lat columns are selected.
                lat = self.dlg.latitude.currentText()
                long = self.dlg.longitude.currentText()
                fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(lat),float(long)))) """
            
            for d in row2: # Here we convert all datatypes to string to make sure they can be supported by QGIS layer
                if isinstance(d, (list, dict)):
                    row2[i] = str(d)
                i += 1

            fet.setAttributes(row2)
            #self.dlg.jsonText.appendPlainText(str(fet.geometry()))
            #self.dlg.jsonText.appendPlainText(str(fet.attributes()))
            writer.addFeature(fet)
            if writer.hasError() != QgsVectorFileWriter.NoError:
                self.dlg.jsonText.appendPlainText("Error when creating file: " + writer.errorMessage())
            else:
                pass
                #self.dlg.jsonText.appendPlainText("Feature written successfully")

        # delete the writer to flush features to disk
        del writer
        
        #self.updateLayer2()
        #if filePath.featureCount() == 0:
            #self.dlg.jsonText.appendPlainText("No features created")

    def createCSV(self, path, filename):
        file = os.path.join(os.path.dirname(path), filename + ".csv")
        responsejson = data.json()
        with open(file, "w", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=self.keysList)

            writer.writeheader()
            for row in responsejson:
                writer.writerow(row)
        self.dlg.outputFile.clear()
        self.dlg.outputFile.setText(file)

    """ def addPcodesColumn(self):
        pass """

    def closeWindow(self):
        self.dlg.close() # Close the mainWindow. The close() function is an inbuilt function in pyqt
    
    def browser(self):
        # then make a url variable
        url = "https://demevengderrick.gitbook.io/odkconnector/"
        # then call the default open method described above
        webbrowser.open(url)
    
    def layerUpdateState(self):
        #toggle the no-geometry combobox state
        if self.dlg.updateLayer.isChecked():
            # self.dlg.outputFile.setEnabled(False)
            # self.dlg.browsOutput.setEnabled(False)
            # self.dlg.addPcodes.setEnabled(False)
            # self.dlg.noGeometry.setEnabled(False)
            self.dlg.layerToUpdate.setEnabled(True)
        else:
            # self.dlg.outputFile.setEnabled(True)
            # self.dlg.browsOutput.setEnabled(True)
            # self.dlg.addPcodes.setEnabled(True)
            # self.dlg.noGeometry.setEnabled(True)
            self.dlg.layerToUpdate.setEnabled(False)

    def updateLayer(self, selectedLayerToUpdate):
        # get the selected layer to be updated
        self.dlg.jsonText.appendPlainText("Update started")
        #self.dlg.jsonText.appendPlainText("Layer Selected")
        geomCol = self.dlg.geometry.currentText()
        #create a list to store coordnates of lines
        lines_coordinates = list() # coordinates of the multi-lines
        line_coordinates = list() # coordinates of one line
        points = []
        polygons = []
        #geomCol = '_geolocation'
        index = self.keysList.index(geomCol)
        selectedLayer = selectedLayerToUpdate
        caps = selectedLayer.dataProvider().capabilities() # get the supported capabilities of the dataProvider (e.g ESRI Shapefile)
        # delete the elements in the layer
        if caps & QgsVectorDataProvider.DeleteFeatures:
            #self.dlg.jsonText.appendPlainText("Delete capabilities supported")
            #check is layer is not in edit mode
            if not selectedLayer.isEditable():
                selectedLayer.startEditing() # start layer editing
                for feat in selectedLayer.getFeatures():
                    selectedLayer.deleteFeature(feat.id())
                    #self.dlg.jsonText.appendPlainText(str(feat.id()))
                selectedLayer.commitChanges() # save edits
            else:
                for feat in selectedLayer.getFeatures():
                    selectedLayer.deleteFeature(feat.id())
                    #self.dlg.jsonText.appendPlainText(str(feat.id()))
        #self.dlg.jsonText.appendPlainText("Data deleted")
        # append new elements into the layer
        if caps & QgsVectorDataProvider.AddFeatures:
            fields = selectedLayer.fields()
            # feature object
            fet = QgsFeature(fields)
            for row in self.dataList:
                i = 0
                row2 = row.copy()
                lenIndex = ""

                if isinstance(row[index], list):
                    if isinstance(row[index][0], dict):
                        if '{}/geotrace'.format(geomCol) in row[index][0].keys():

                            #self.dlg.jsonText.appendPlainText("Nothing to save yet")
                            #geomCol = "segment"
                            #index = self.keysList.index(geomCol)
                            #self.dlg.jsonText.appendPlainText(row[index][1])
                            for k in row[index]:
                                for j in k['{}/geotrace'.format(geomCol)].split(";"):
                                    j = j.strip() # Remove leading/trailing spaces
                                    if j:  # Check if the string is not empty
                                        # Create multi-line geometry from the list of coordinate
                                        coordinates = j.split(" ") #long, lat, alt, precision
                                        line_coordinates.append((coordinates[0], coordinates[1]))
                                        #self.dlg.jsonText.appendPlainText("reached here")
                                        #self.dlg.jsonText.appendPlainText(j.split(" ")[0])
                                lines_coordinates.append(line_coordinates)
                                line_coordinates = list() # reset the list for a new line

                            multi_line = QgsGeometry.fromMultiPolylineXY([[QgsPointXY(float(y), float(x)) for x, y in line] for line in lines_coordinates])  
                            fet.setGeometry(multi_line)
                            lines_coordinates = list() # reset the list for new coordinates
                            #self.dlg.jsonText.appendPlainText("Check 8")
                        
                        elif '{}/geoshape'.format(geomCol) in row[index][0].keys():
                            for k in row[index]:
                                coordinates = k['segment/geoshape'].split(';')
                                points = [QgsPointXY(float(coord.split()[1]), float(coord.split()[0])) for coord in coordinates]
                                polygons.append([points])  # Each polygon is a list of points
                            
                            # Create a multipolygon feature from the polygons
                            multipolygon = QgsGeometry.fromMultiPolygonXY(polygons)
                            fet.setGeometry(multipolygon)
                            polygons = []

                        elif '{}/geopoint'.format(geomCol) in row[index][0].keys():
                            for k in row[index]:
                                lat, lon = map(float, k['segment/geopoint'].split()[:2])
                                points.append(QgsPointXY(lon, lat))

                            # Create a multipoint feature from the points
                            multipoint = QgsGeometry.fromMultiPointXY(points)
                            fet.setGeometry(multipoint)
                            points = []
                    else:
                        #self.dlg.jsonText.appendPlainText("Check 13")
                        fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(row[index][1]),float(row[index][0]))))
                        #self.dlg.jsonText.appendPlainText("I was here before.................LOL")
                        #self.dlg.jsonText.appendPlainText(str(float(row[index][1])))
                        

                elif lenIndex > 1: # check if the feature is a point, line or polygone
                    split = row[index].split(";")
                    #self.dlg.jsonText.appendPlainText("Check 9")
                    if split[0] == split[-1]:
                        #self.dlg.jsonText.appendPlainText("Check 10")
                        points = [QgsPointXY(float(c.split()[1]), float(c.split()[0])) for c in split]
                        poly = QgsGeometry.fromPolygonXY([points])
                        fet.setGeometry(poly)
                        #self.dlg.jsonText.appendPlainText("Check 11")
                    else:
                        #self.dlg.jsonText.appendPlainText("Check 12")
                        self.dlg.jsonText.appendPlainText(str(split))
                        points = [QgsPointXY(float(c.split()[1]), float(c.split()[0])) for c in split]
                        line = QgsGeometry.fromPolylineXY(points)
                        fet.setGeometry(line)

                elif lenIndex == 1:
                    #self.dlg.jsonText.appendPlainText("Check 14")
                    self.dlg.jsonText.appendPlainText(row[index])
                    fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(row[index].split(" ")[1]),float(row[index].split(" ")[0]))))

                """ else: # do when lon and lat columns are selected.
                    lat = self.dlg.latitude.currentText()
                    long = self.dlg.longitude.currentText()
                    fet.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(float(lat),float(long)))) """
                
                for d in row2: # Here we convert all datatypes to string to make sure they can be supported by QGIS layer
                    if isinstance(d, (list, dict)):
                        row2[i] = str(d)
                        #row2.insert(0, i+1)
                    i += 1

                fet.setAttributes(row2)
                selectedLayer.dataProvider().addFeature(fet)
            
        selectedLayer.commitChanges()
        self.dlg.jsonText.appendPlainText("Data Updated")
    
    def changeButtonText(self):
        if self.dlg.tabWidget.currentIndex() == 0:
            self.dlg.runProcess.setText("Run")
        else:
            self.dlg.runProcess.setText("Update Layer")

    def process(self):
        try:
            self.dlg.progressBar.setValue(0)
            crs = self.dlg.csr.crs()
            self.dlg.processingMsg.setText("Processing...")
            self.dlg.progressBar.setValue(10)
            selectedLayer = self.dlg.layerToUpdate.currentLayer()
            if self.dlg.updateLayer.isChecked():
                pass
            else:
                outFile = self.dlg.outputFile.text() #output file path
                fileName = os.path.basename(outFile).split(".")[0]#output path basename
                format = os.path.basename(outFile).split(".")[1]
                self.dlg.progressBar.setValue(20)

            if self.dlg.noGeometry.isChecked():#if no geometry is required or if file has no geometry
                self.createCSV(outFile, fileName)
                self.dlg.progressBar.setValue(80)

            elif self.dlg.updateLayer.isChecked():
                self.updateLayer(selectedLayer)
                self.dlg.progressBar.setValue(50)
            else:
                self.createFeatureClass(outFile, format, crs)
                #self.dlg.jsonText.appendPlainText(str(QgsVectorLayer(outFile, fileName, "ogr").featureCount()))
                """ if QgsVectorLayer(outFile, fileName, "ogr").featureCount() == 0: 
                    # for some strange reason, the writer sometimes fail to write features to the layer//
                    # so if this happens we use the update function to rewrite the features and for some strange reason again it works
                    #self.dlg.jsonText.appendPlainText("No features in the layer")
                    selectedLayer = QgsVectorLayer(outFile, fileName, "ogr")
                    self.updateLayer(selectedLayer)
                self.dlg.progressBar.setValue(80) """

            if self.dlg.loadData.isChecked() and self.dlg.updateLayer.isChecked() == False:
                vlayer = QgsVectorLayer(outFile, fileName, "ogr") #QGIS vector data loader
                QgsProject.instance().addMapLayer(vlayer) # adding the vector data into QGIS
                self.dlg.progressBar.setValue(90)
            self.dlg.processingMsg.setText('<span style=\" color: #006400;\">%s</span>' % ("Done!"))
            self.dlg.progressBar.setValue(100)

        except Exception as e:
            self.dlg.processingMsg.setText('<span style=\" color: #ff0000;\">%s</span>' % str(e))
            self.dlg.progressBar.setValue(0)

""" 
In this section we are going to handle automatic refresh of layers
"""
""" # Function to control the timer based on the checkbox state
def on_timer_timeout(self):
    selectedLayer = self.dlg.layerToUpdate.currentLayer()
    self.updateLayer(selectedLayer)  # Start the timer with an interval of 1 minute

def start_auto_update(self, interval):
    self.timer.start(interval)

def stop_auto_update(self):
    self.timer.stop()

def toggle_auto_update(self):
    if self.dlg.updateLayer.isChecked():
        interval = self.dlg.refreshTime.value() * 60000  # Convert seconds to milliseconds
        self.start_auto_update(interval)
    else:
        self.stop_auto_update()

def change_update_interval(self, value):
    if self.dlg.autoUpdateCheckbox.isChecked():
        self.start_auto_update(value * 1000)  # Restart timer with new interval """