Hssetup

.devcontainer/Dockerfile

@@ -11,4 +11,6 @@ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bi
 
 RUN apt-get update && apt-get -y install --no-install-recommends \
   build-essential cmake libtool-bin git less wget \
-  libhdf5-dev libhdf5-mpi-dev hdf5-tools libyajl-dev
+  libhdf5-dev libhdf5-mpi-dev hdf5-tools libyajl-dev python3-pip
+RUN conda install -n base h5py matplotlib h5netcdf xarray
+RUN pip3 install h5pyd s3fs

.devcontainer/devcontainer.json

@@ -1,10 +1,16 @@
 {
   "name": "HDF5 Tutorial",
-  "build": {
-    "context": "..",
-    "dockerfile": "Dockerfile"
-  },
+  "dockerComposeFile": "docker-compose.yaml",
   "postCreateCommand": "conda init",
+  "service": "app",
+	"workspaceFolder": "/workspace",
+	"forwardPorts": [5101],
+	"portsAttributes": {
+		"5101": {"label": "HSDS port", "onAutoForward": "silent"}
+	},
+  "features": {
+    "ghcr.io/devcontainers/features/docker-outside-of-docker": {}
+  },
   "customizations": {
     "vscode": {
       "extensions": [
@@ -23,4 +29,4 @@
       }
     }
   }
-}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,120 @@
+version: "3"
+services:
+  app:
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+    environment:
+      - HS_ENDPOINT=http://localhost:5101
+      - HS_USERNAME=vscode
+      - HS_PASSWORD=vscode
+    volumes:
+      - ..:/workspace:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the SN container, allows "forwardPorts" in devcontainer.json function.
+    network_mode: service:sn
+  head:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 512m
+    environment:
+      - TARGET_SN_COUNT=1
+      - TARGET_DN_COUNT=4
+      - NODE_TYPE=head_node
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+    ports:
+      - 5100
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+  dn1:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+    ports:
+      - 6101
+    depends_on:
+      - head
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+    links:
+      - head
+  dn2:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+    ports:
+      - 6102
+    depends_on:
+      - head
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+    links:
+      - head
+  dn3:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+    ports:
+      - 6103
+    depends_on:
+      - head
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+    links:
+      - head
+  dn4:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - NODE_TYPE=dn
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+    ports:
+      - 6104
+    depends_on:
+      - head
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+    links:
+      - head
+  sn:
+    image: hdfgroup/hsds:master
+    restart: on-failure
+    mem_limit: 1g
+    environment:
+      - SN_PORT=5101
+      - NODE_TYPE=sn
+      - ROOT_DIR=/data
+      - BUCKET_NAME=hsdstest
+      - HSDS_ENDPOINT=http://localhost:5101
+    ports:
+      - 5101:5101
+    depends_on:
+      - head
+    volumes:
+      - ../hsds/data:/data
+      - ../hsds/config/:/config/
+    links:
+      - head

05-HSDS.ipynb

@@ -10,45 +10,13 @@
     "\n",
     "## Installation\n",
     "\n",
-    "For this simplistic setup, the installation is easy:\n",
+    "This code space is already configured to run HSDS, so no special setup is needed. \n",
+    "Refer to the files under .vscode in this project to see how it is done.\n",
     "\n",
-    "1. Create a directory for the HSDS data files (`~/hsds_data`).\n",
-    "2. Use the user name `vscode` and password `vscode` to authenticate to HSDS.\n",
-    "3. Launch the service.\n",
-    "\n",
-    "For more sophisticated setups (e.g., Kubernetes), please refer to the [HSDS documentation](https://github.com/HDFGroup/hsds/tree/master/docs)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%%bash\n",
-    "export HS_ENDPOINT=http://localhost:5101\n",
-    "export HS_USERNAME=$USER\n",
-    "export HS_PASSWORD=$USER\n",
-    "mkdir ~/hsds_data\n",
-    "hsds --root_dir ~/hsds_data --hs_username $USER --hs_password $USER >~/hs.log 2>&1 &\n",
-    "sleep 5"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We create a configuration file for HSDS: `~/.hscfg`:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%%bash\n",
-    "hsconfigure <<< $'http://localhost:5101\\nvscode\\nvscode\\n\\nY\\n'"
+    "Here HSDS is running in the cloud as part of Github Code Spaces.  You can also installed \n",
+    "on your Mac/Windows/or Linux desktop.\n",
+    "Please refer to the [HSDS documentation](https://github.com/HDFGroup/hsds/tree/master/docs)\n",
+    "for detailed instructions."
    ]
   },
   {
@@ -72,7 +40,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Create the top-level domain and a user \"directory\" for the user `vscode`:"
+    "When you first run the codespace, HSDS will have just a hdf5:/home/ and \n",
+    "hdf5:/home/vscode/ folders."
    ]
   },
   {
@@ -82,8 +51,7 @@
    "outputs": [],
    "source": [
     "%%bash\n",
-    "hstouch /home/ && hstouch /home/$USER/\n",
-    "hsinfo"
+    "hsls hdf5://home/"
    ]
   },
   {
@@ -140,7 +108,7 @@
    "outputs": [],
    "source": [
     "%%bash\n",
-    "curl -s -u vscode:vscode http://localhost:5101/?domain=/home/vscode/foo.h5 | jq"
+    "curl -s http://localhost:5101/?domain=/home/vscode/foo.h5 | jq"
    ]
   },
   {
@@ -150,7 +118,7 @@
    "outputs": [],
    "source": [
     "%%bash\n",
-    "find ~/hsds_data"
+    "find ~/hsds/data/"
    ]
   },
   {
@@ -407,7 +375,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.0"
+   "version": "3.9.18"
   }
  },
  "nbformat": 4,

hsds/config/override.yml

@@ -0,0 +1,13 @@
+log_level: ERROR  # or DEBUG or INFO or WARNING
+bucket_name: hsdstest  # default bucket name
+server_name: HSDS for Github codespaces   # this is displayed by the hsinfo command
+chunk_mem_cache_size: 256m  # chunk_cache memory per DN container
+root_dir: /workspace/hsds/data  # this is where HSDS data files will be stored
+default_public: True  # New domains will be created as default read
+aws_region: us-west-2  # The majority of public data files are in us-west-2 
+aws_s3_gateway: http://s3.us-west-2.amazonaws.com/  # s3 gateway for us-west-2
+aws_s3_no_sign_request: True  # Set to False if setting aws_access_key_id and aws_secret_access_key here 
+head_sleep_time: 60 # max sleep time between health checks for head node
+node_sleep_time: 60 # max sleep time between health checks for SN/DN nodes
+aws_region: us-west-2 # (original was us-east-1)
+aws_s3_gateway: http://s3.us-west-2.amazonaws.com/ # (original was null)

hsds/config/passwd.txt

@@ -0,0 +1,2 @@
+# username/passwords for HSDS...
+vscode:vscode

hsds/data/hsdstest/home/.domain.json

@@ -0,0 +1 @@
+{"owner": "admin", "acls": {"admin": {"create": true, "read": true, "update": true, "delete": true, "readACL": true, "updateACL": true}, "default": {"create": false, "read": true, "update": false, "delete": false, "readACL": false, "updateACL": false}}, "created": 1708897646.0599918, "lastModified": 1708897646.0599918}

hsds/data/hsdstest/home/vscode/.domain.json

@@ -0,0 +1 @@
+{"owner": "vscode", "acls": {"vscode": {"create": true, "read": true, "update": true, "delete": true, "readACL": true, "updateACL": true}, "default": {"create": false, "read": true, "update": false, "delete": false, "readACL": false, "updateACL": false}}, "created": 1711992550.3733413, "lastModified": 1711992550.3733413}

hsds/data/hsdstest/readme.txt

@@ -0,0 +1 @@
+This is the default bucket for HSDS

hsds/data/readme.txt

@@ -0,0 +1 @@
+This is the root directory for HSDS data

python/01-Intro.ipynb

@@ -24,13 +24,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "USE_H5PY = True  # set to False to use HSDS instead\n",
+    "USE_H5PY = False  # set to False to use HSDS instead\n",
     "if USE_H5PY:\n",
     "    import h5py\n",
-    "    WORK_DIR=\".\"  # this directory\n",
+    "    WORK_DIR = \".\"\n",
     "else:\n",
     "    import h5pyd as h5py\n",
-    "    WORK_DIR=\"hdf5://home/test_user1/\"\n",
+    "    WORK_DIR = \"hdf5://home/vscode/\"\n",
     "import os.path as op"
    ]
   },
@@ -187,7 +187,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.1"
+   "version": "3.9.15"
   }
  },
  "nbformat": 4,