Merge pull request #4 from brandenm-nag/fs_fixup

Filestore creation fixes, VPC and Location fixes

frontend/website/ghpcfe/cluster_manager/cloud_info.py

@@ -405,3 +405,22 @@ def get_gcp_workbench_region_zone_info(credentials, service="notebooks", api_ver
     locations = [x['locationId'] for x in result['locations']]
     return locations
 
+def get_gcp_filestores(credentials):
+    """Returns an array of Filestore instance informations
+    [
+      {'creatTime': ...,
+       'fileShares': [{'capacityGb': '2660', 'name': 'data'}],
+       'name': 'projects/<project>/locations/<zone>/instances/<name>',
+       'networks': [{'ipAddresses': ['10.241.201.242'], 'modes': ['MODE_IPV4'], 'network': '<network-name>', 'reservedIpRange': '10.241.201.240/29'}],
+       'state': 'READY',
+       'tier': 'PREMIUM'
+      },
+      ...
+    ]
+    """
+    (project, client) = _get_gcp_client(credentials, "file", "v1");
+    request = client.projects().locations().instances().list(parent=f"projects/{project}/locations/-")
+    result = request.execute()
+    return result['instances']
+
+

frontend/website/ghpcfe/cluster_manager/filesystem.py

@@ -51,7 +51,7 @@ def write_filestore_yaml(fs: GCPFilestoreFilesystem, tgtDir: Path) -> None:
     id: {fs.name}
     settings:
       filestore_share_name: {export_name[1:]}
-      network_name: {fs.subnet.vpc.cloud_id}
+      network_name: {fs.vpc.cloud_id}
       zone: {fs.cloud_zone}
       size_gb: {fs.capacity}
       filestore_tier: {fs.get_performance_tier_display()}
@@ -85,7 +85,7 @@ def create_filesystem(fs: Filesystem) -> None:
 
 
 def _run_ghpc(tgtDir: Path) -> None:
-    ghpc_path = utils.load_config()["baseDir"] / 'dependencies' / 'hpc-toolkit' / 'ghpc'
+    ghpc_path = utils.load_config()["baseDir"].parent / 'ghpc'
 
     try:
         logger.info("Invoking ghpc create")

frontend/website/ghpcfe/forms.py

@@ -21,6 +21,7 @@
 from django.utils.safestring import mark_safe
 from .cluster_manager import validate_credential, cloud_info
 from .models import *
+import itertools
 import json
 
 import logging
@@ -457,30 +458,32 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         creds = self._get_creds(kwargs)
+        self.fields['vpc'].queryset = VirtualNetwork.objects.filter(cloud_credential=creds).filter(Q(cloud_state="i")|Q(cloud_state="m"))
+        region_info = cloud_info.get_region_zone_info("GCP", creds.detail)
+        self.fields['cloud_zone'].widget.choices = [(r, [(z, z) for z in rz]) for r,rz in region_info.items()]
 
-        self.fields['subnet'].queryset = VirtualSubnet.objects.filter(cloud_credential=creds).filter(Q(cloud_state="i")|Q(cloud_state="m"))
         if zone_choices:
             # We set this on the widget, because we will be changing the
             # widget's field in the template via javascript
             self.fields['cloud_zone'].widget.choices = zone_choices
 
         if 'n' not in self.instance.cloud_state:
             # Need to disable certain widgets
-            self.fields['subnet'].disabled = True
+            self.fields['vpc'].disabled = True
             self.fields['cloud_zone'].disabled = True
             self.fields['share_name'].disabled = True
             self.fields['performance_tier'].disabled = True
 
     class Meta:
         model = GCPFilestoreFilesystem
 
-        fields = ('name', 'subnet', 'cloud_zone', 'cloud_credential', 'capacity', 'performance_tier')
+        fields = ('name', 'vpc', 'cloud_zone', 'cloud_credential', 'capacity', 'performance_tier')
 
         widgets = {
             'name': forms.TextInput(attrs={'class': 'form-control'}),
             'cloud_credential': forms.Select(attrs={'class': 'form-control', 'disabled': True}),
-            'subnet': forms.Select(attrs={'class': 'form-control'}), 
-            'capacity': forms.NumberInput(attrs={'min': 1024, 'default':1024}),
+            'vpc': forms.Select(attrs={'class': 'form-control'}), 
+            'capacity': forms.NumberInput(attrs={'min': 2660, 'default':2660}),
             'share_name': forms.TextInput(attrs={'class': 'form-control'}),
             'cloud_zone': forms.Select(attrs={'class': 'form-control'}),
         }

frontend/website/ghpcfe/models.py

@@ -298,8 +298,16 @@ class Filesystem(CloudResource):
     subnet = models.ForeignKey(
         VirtualSubnet,
         related_name = 'filesystems',
-        help_text = 'Subnet within which the cluster resides',
+        help_text = 'Subnet within which the Filesystem resides (if any)',
         on_delete = models.RESTRICT,
+        null = True,
+    )
+    vpc = models.ForeignKey(
+        VirtualNetwork,
+        related_name = 'filesystems',
+        help_text = 'Network within which the Filesystem resides',
+        on_delete = models.SET_NULL,
+        null = True,
     )
     impl_type = models.PositiveIntegerField(
         choices = FilesystemImpl.choices,
@@ -903,7 +911,7 @@ class GCPFilestoreFilesystem(Filesystem):
     )
     capacity = models.PositiveIntegerField(
         validators = [MinValueValidator(1024)],
-        help_text = 'Capacity (in GB) of the filesystem (min of 1024)',
+        help_text = 'Capacity (in GB) of the filesystem (min of 2660)',
         default = 1024
     )
 

frontend/website/ghpcfe/templates/filesystem/filestore_create_form.html

@@ -1,30 +1,6 @@
 {% extends "base_generic.html" %}
 
 {% block extrameta %}
-<script>
-function subnetSelected() {
-    {% autoescape off %}
-    var subnet_map = {{ subnet_regions }};
-    var region_info = {{ region_info }};
-    {% endautoescape %}
-    subnet_element = document.getElementById("id_subnet");
-    zone_element = document.getElementById("id_cloud_zone");
-    zone_element.disabled = false;
-    $("#id_cloud_zone").find('option').remove().end();
-    region_info[subnet_map[subnet_element.value]].forEach(function (item) {
-      var el = document.createElement("option");
-      el.text = item;
-      el.setAttribute("value", item) 
-      zone_element.appendChild(el);
-    });
-}
-
-function initPage() {
-    document.getElementById("id_subnet").onchange = subnetSelected;
-}
-
-window.onload=initPage;
-</script>
 {% endblock %}
 
 
@@ -54,10 +30,10 @@ <h2>Create a new GCP Filestore</h2>
         <small class="form-text text-muted">{{ form.cloud_credential.help_text }}</small>
     </div>
     <div class="form-group">
-      {{ form.subnet.errors }}
-      {{ form.subnet.label_tag }}
-      {{ form.subnet }}
-        <small class="form-text text-muted">{{ form.subnet.help_text }}</small>
+      {{ form.vpc.errors }}
+      {{ form.vpc.label_tag }}
+      {{ form.vpc }}
+        <small class="form-text text-muted">{{ form.vpc.help_text }}</small>
     </div>
     <div class="form-group">
       {{ form.cloud_zone.errors }}

frontend/website/ghpcfe/templates/filesystem/filestore_update_form.html

@@ -1,30 +1,6 @@
 {% extends "base_generic.html" %}
 
 {% block extrameta %}
-<script>
-function subnetSelected() {
-    {% autoescape off %}
-    var subnet_map = {{ subnet_regions }};
-    var region_info = {{ region_info }};
-    {% endautoescape %}
-    subnet_element = document.getElementById("id_subnet");
-    zone_element = document.getElementById("id_cloud_zone");
-    zone_element.disabled = false;
-    $("#id_cloud_zone").find('option').remove().end();
-    region_info[subnet_map[subnet_element.value]].forEach(function (item) {
-      var el = document.createElement("option");
-      el.text = item;
-      el.setAttribute("value", item) 
-      zone_element.appendChild(el);
-    });
-}
-
-function initPage() {
-    document.getElementById("id_subnet").onchange = subnetSelected;
-}
-
-window.onload=initPage;
-</script>
 {% endblock %}
 
 
@@ -54,10 +30,10 @@ <h2>Edit a GCP Filestore</h2>
         <small class="form-text text-muted">{{ form.cloud_credential.help_text }}</small>
     </div>
     <div class="form-group">
-      {{ form.subnet.errors }}
-      {{ form.subnet.label_tag }}
-      {{ form.subnet }}
-        <small class="form-text text-muted">{{ form.subnet.help_text }}</small>
+      {{ form.vpc.errors }}
+      {{ form.vpc.label_tag }}
+      {{ form.vpc }}
+        <small class="form-text text-muted">{{ form.vpc.help_text }}</small>
     </div>
     <div class="form-group">
       {{ form.cloud_zone.errors }}

frontend/website/ghpcfe/templates/filesystem/list.html

@@ -27,7 +27,7 @@ <h2>Filesystem List</h2>
         <th scope="col">Name</th>
         <th scope="col">Type</th>
         <th scope="col">Cloud ID</th>
-        <th scope="col">Region</th>
+        <th scope="col">Zone</th>
         <th scope="col">Status</th>
         <th scope="col">Actions</th>
       </tr>
@@ -39,7 +39,7 @@ <h2>Filesystem List</h2>
         <td><a href="{% url 'fs-detail' fs.id %}">{{ fs.name }}</a></td>
         <td>{{ fs.get_impl_type_display }}</td>
         <td>{{ fs.cloud_id }}</td>
-        <td>{{ fs.cloud_region }}</td>
+        <td>{{ fs.cloud_zone }}</td>
         <td>
           {% if "c" in fs.cloud_state == "c" %}
           <img src="/static/img/loading.gif" style="width:32px;height:32px;">

frontend/website/ghpcfe/templates/vpc/list.html

@@ -97,15 +97,14 @@ <h2>Virtual Network List</h2>
 <script language="javascript">
 function filterTable() {
     var toggle = document.getElementById("id_show_destroyed").checked;
-    console.log(toggle);
     $.cookie("vpc_list_show_destroyed", toggle, {path: "{% url 'vpcs' %};SameSite=Strict", expires: 10});
     var table = document.getElementById("id_vpc_table");
     var tr = table.getElementsByTagName("tbody")[0].getElementsByTagName("tr");
 
     for ( i = 0 ; i < tr.length; i++ ) {
-        var td = tr[i].getElementsByTagName("td")[3];
+        var td = tr[i].getElementsByTagName("td")[2];
         var tdText = td.textContent || td.innerText;
-        if ( toggle || tdText.indexOf("has been deleted") == -1 ) {
+        if ( toggle || tdText.indexOf("Destroyed") == -1 ) {
             tr[i].style.display = "";
         } else {
             tr[i].style.display = "none";

frontend/website/ghpcfe/views/clusters.py

@@ -215,7 +215,7 @@ def formfield_cb(modelField, **kwargs):
                 fsquery = Filesystem.objects    \
                             .exclude(impl_type=FilesystemImpl.BUILT_IN) \
                             .filter(cloud_state__in=['m', 'i']) \
-                            .filter(subnet__vpc=cluster.subnet.vpc).values_list('pk', flat=True)
+                            .filter(vpc=cluster.subnet.vpc).values_list('pk', flat=True)
                 # Add back our cluster's filesystem
                 fsystems = list(fsquery) + [cluster.shared_fs.id]
                 field.queryset = FilesystemExport.objects.filter(filesystem__in=fsystems)
@@ -286,8 +286,7 @@ def form_valid(self, form):
         # Verify formset validity (suprised there's not another method to do this)
         for formset in [mountpoints, partitions]:
             if not formset.is_valid():
-                for error in formset.errors:
-                    form.add_error(None, error)
+                form.add_error(None, "Error in form below")
                 return self.form_invalid(form)
 
         with transaction.atomic():
@@ -300,6 +299,12 @@ def form_valid(self, form):
         if (self.object.status == 'r'):
             msg = "Cluster configuration updated. Click 'Edit' button again to make further changes and click 'Sync Cluster' button to apply changes."
         messages.success(self.request, msg)
+
+        # Be kind... Check filesystems to verify all in the same zone as us.
+        for mp in self.object.mount_points.exclude(export__filesystem__impl_type=FilesystemImpl.BUILT_IN):
+            if mp.export.filesystem.cloud_zone != self.object.cloud_zone:
+                messages.warning(self.request, f"Possibly expensive: Filesystem {mp.export.filesystem.name} is in a different zone ({mp.export.filesystem.cloud_zone}) than the cluster!")
+
         return super().form_valid(form)
 
 

frontend/website/ghpcfe/views/filesystems.py

@@ -191,6 +191,8 @@ def get_orm(self, fs_id):
         return (fs,)
 
     def cmd(self, task_id, token, fs):
+        fs.cloud_state = 'cm'
+        fs.save()
         cm_fs.create_filesystem(fs)
 
     async def get(self, request, pk):

frontend/website/ghpcfe/views/gcpfilestore.py

@@ -61,31 +61,26 @@ class GCPFilestoreFilesystemUpdateView(UpdateView):
     template_name = 'filesystem/filestore_update_form.html'
     form_class = FilestoreForm
 
-    def _get_region_info(self):
-        if not hasattr(self, 'region_info'):
-            self.region_info = cloud_info.get_region_zone_info("GCP", self.get_object().cloud_credential.detail)
-        return self.region_info
-
     def get_success_url(self):
         return reverse('backend-filesystem-update-files', kwargs={'pk': self.object.pk})
 
     def get_initial(self):
-        return {'share_name': self.get_object().exports.all()[0].export_name}
+        return {'share_name': self.get_object().exports.first().export_name}
 
-    def get_form_kwargs(self):
-        kwargs = super().get_form_kwargs()
-        kwargs['zone_choices'] = [(x,x) for x in self._get_region_info()[self.get_object().cloud_region]]
-        return kwargs
+    def form_valid(self, form):
+        self.object = form.save(commit=False)
+        self.object.cloud_region = self.object.cloud_zone.rsplit('-', 1)[0]
+        self.object.impl_type = FilesystemImpl.GCPFILESTORE
+        self.object.save()
 
+        export = self.object.exports.first()
+        export.export_name = form.data['share_name']
+        export.save()
 
-    def get_context_data(self, **kwargs):
-        """ Perform extra query to populate instance types data """
-        subnet_regions = {sn.id: sn.cloud_region for sn in VirtualSubnet.objects.filter(cloud_credential=self.get_object().cloud_credential).filter(Q(cloud_state="i") | Q(cloud_state="m")).all()}
+        return HttpResponseRedirect(self.get_success_url())
 
+    def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-
-        context['subnet_regions'] = json.dumps(subnet_regions)
-        context['region_info'] = json.dumps(self._get_region_info())
         context['navtab'] = 'fs'
         return context
 
@@ -98,12 +93,6 @@ class GCPFilestoreFilesystemCreateView(LoginRequiredMixin, generic.CreateView):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-
-        self.region_info = cloud_info.get_region_zone_info("GCP", self.cloud_credential.detail)
-        subnet_regions = {sn.id: sn.cloud_region for sn in VirtualSubnet.objects.filter(cloud_credential=self.cloud_credential).filter(Q(cloud_state="i") | Q(cloud_state="m")).all()}
-
-        context['subnet_regions'] = json.dumps(subnet_regions)
-        context['region_info'] = json.dumps(self.region_info)
         context['navtab'] = 'fs'
         return context
 
@@ -120,7 +109,7 @@ def post(self, request, *args, **kwargs):
 
     def form_valid(self, form):
         self.object = form.save(commit=False)
-        self.object.cloud_region = self.object.subnet.cloud_region;
+        self.object.cloud_region = self.object.cloud_zone.rsplit('-', 1)[0]
         self.object.impl_type = FilesystemImpl.GCPFILESTORE
         self.object.save()