LakeFS

docker run --name lakefs --rm \
             --publish 8000:8000 \
             treeverse/lakefs:latest \
             run --local-settings

version: "3.5"
name: lakefs-example
services:
  lakefs:
    image: treeverse/lakefs:1
    pull_policy: always
    ports:
      - "8000:8000"
    depends_on:
      postgres:
        condition: service_healthy
      minio-setup:
        condition: service_completed_successfully
    environment:
      - LAKEFS_BLOCKSTORE_TYPE=s3
      - LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true
      - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://minio:9000
      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=minioadmin
      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=minioadmin
      - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
      - LAKEFS_LOGGING_LEVEL=INFO
      - LAKEFS_STATS_ENABLED=${LAKEFS_STATS_ENABLED:-1}
      - LAKECTL_CREDENTIALS_ACCESS_KEY_ID=AKIAIOSFOLKFSSAMPLES
      - LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
      - LAKECTL_SERVER_ENDPOINT_URL=http://localhost:8000
      - LAKEFS_DATABASE_TYPE=postgres
      - LAKEFS_DATABASE_POSTGRES_CONNECTION_STRING=postgres://postgres:postgres@postgres/postgres?sslmode=disable
    entrypoint: ["/bin/sh", "-c"]
    command:
        - |
          lakefs setup --user-name everything-bagel --access-key-id "$$LAKECTL_CREDENTIALS_ACCESS_KEY_ID" --secret-access-key "$$LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY" || true
          lakefs run &
          echo "---- Creating repository ----"
          wait-for -t 60 lakefs:8000 -- curl -u "$$LAKECTL_CREDENTIALS_ACCESS_KEY_ID":"$$LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY" -X POST -H "Content-Type: application/json" -d '{ "name": "quickstart", "storage_namespace": "s3://quickstart", "default_branch": "main", "sample_data": true }' http://localhost:8000/api/v1/repositories || true
          # wait-for -t 60 lakefs:8000 -- lakectl repo create lakefs://example s3://example || true
          echo ""
          wait-for -t 60 minio:9000 && echo '------------------------------------------------

                MinIO admin:   http://127.0.0.1:9001/
                
                               Username : minioadmin
                               Password : minioadmin
                '
          echo "------------------------------------------------"
          echo ""
          echo "lakeFS Web UI: http://127.0.0.1:8000/      >(.＿.)<"
          echo "                                             (  )_ "
          echo ""
          echo "                Access Key ID    : $$LAKECTL_CREDENTIALS_ACCESS_KEY_ID"
          echo "                Secret Access Key: $$LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY"
          echo ""
          echo "-------- Let's go and have axolotl fun! --------"
          echo ""
          wait
  minio-setup:
    image: minio/mc:RELEASE.2023-05-18T16-59-00Z
    environment:
        - MC_HOST_lakefs=http://minioadmin:minioadmin@minio:9000
    depends_on:
      - minio
    volumes:
      - ./data:/data
    entrypoint: ["/bin/sh", "-c"]
    command:
        - |
          mc mb lakefs/quickstart lakefs/example lakefs/sample-data
          exit 0

  minio:
    image: minio/minio:RELEASE.2023-05-18T00-05-36Z
    ports:
      - "9000:9000"
      - "9001:9001"
    entrypoint: ["minio", "server", "/data", "--console-address", ":9001"]

  postgres:
    image: postgres:14
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "postgres"]
      interval: 10s
      retries: 5
      start_period: 5s
    restart: always 

networks:
  default:
    name: bagel

import boto3
s3 = boto3.client('s3',
    endpoint_url='https://lakefs.example.com',
    aws_access_key_id='AKIAIOSFODNN7EXAMPLE',
    aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY')

with open('/local/path/to/file_0', 'rb') as f:
    s3.put_object(Body=f, Bucket='example-repo', Key='main/example-file.parquet')

list_resp = s3.list_objects_v2(Bucket='example-repo', Prefix='c7a632d74f/example-prefix')
for obj in list_resp['Contents']:
    print(obj['Key'])

s3.head_object(Bucket='example-repo', Key='main/example-file.parquet')

from lakefs.client import Client

clt = Client(
    host="http://localhost:8000",
    username="AKIAIOSFODNN7EXAMPLE",
    password="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
)

repo = lakefs.Repository("example-repo", client=clt).create(storage_namespace="s3://storage-bucket/repos/example-repo")

branch1 = lakefs.repository("example-repo").branch("experiment1").create(source_reference_id="main")

obj = branch1.object(path="text/sample_data.txt").upload(content_type="text/plain", data="This is my object data")
print(obj.stats())

print(obj.reader(mode='r').read())

obj = branch1.object(path="raw/file1.data").upload(data=b"Hello Object World", pre_sign=True)
print(obj.stats())

for diff in branch1.uncommitted():
    print(diff)

ref = branch1.commit(message='Add some data!', metadata={'using': 'python_sdk'})
print(ref.get_commit())

main = repo.branch("main")
for diff in main.diff(other_ref=branch1):
    print(diff)

res = branch1.merge_into(main)
print(res)

from pathlib import Path
from lakefs_spec import LakeFSFileSystem

REPO, BRANCH = "repo", "main"

# Prepare example local data
local_path = Path("demo.txt")
local_path.write_text("Hello, lakeFS!")

# Upload the local file to the repo and commit
fs = LakeFSFileSystem()  # will auto-discover credentials from ~/.lakectl.yaml
repo_path = f"{REPO}/{BRANCH}/{local_path.name}"
with fs.transaction(REPO, BRANCH) as tx:
    fs.put(str(local_path), f"{REPO}/{tx.branch.id}/{local_path.name}")
    tx.commit(message="Add demo data")

# Read back the file contents
f = fs.open(repo_path, "rt")
print(f.readline())  # prints "Hello, lakeFS!"

# Compare the sizes of local file and repo
file_info = fs.info(repo_path)
print(
    f"{local_path.name}: local size: {file_info['size']}, remote size: {local_path.stat().st_size}"
)

# Get information about all files in the repo root
print(fs.ls(f"{REPO}/{BRANCH}/"))

# Delete uploaded file from the repository (and commit)
with fs.transaction(REPO, BRANCH) as tx:
    fs.rm(f"{REPO}/{tx.branch.id}/{local_path.name}")
    tx.commit(message="Delete demo data")

local_path.unlink()

$ lakectl local clone lakefs://is-alpaca/experiment-1/dataset/train/ input

Successfully cloned lakefs://is-alpaca/experiment-1/dataset/train/ to ~/ml_models/is_alpaca/input

Clone Summary:

Downloaded: 250
Uploaded: 0
Removed: 0

is_alpaca$ lakectl local list                 
+-----------+------------------------------------------------+------------------------------------------------------------------+
| DIRECTORY | REMOTE URI                                     | SYNCED COMMIT                                                    |
+-----------+------------------------------------------------+------------------------------------------------------------------+
| input     | lakefs://is-alpaca/experiment-1/dataset/train/ | 589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3 |
+-----------+------------------------------------------------+------------------------------------------------------------------+

is_alpaca$ git add input/
is_alpaca$ git status 
On branch experiment-1
Changes to be committed:
  (use "git restore --staged <file>..." to unstage)
    new file:   input/.lakefs_ref.yaml

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
    modified:   .gitignore

is_alpaca$ cat input/.lakefs_ref.yaml

src: lakefs://is-alpaca/experiment-1/dataset/train
at_head: 589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3

is_alpaca$ git commit -m "added is_alpaca dataset" 

is_alpaca$ lakectl local status input 
diff 'local:///ml_models/is_alpaca/input' <--> 'lakefs://is-alpaca/589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3/dataset/train/'...
diff 'lakefs://is-alpaca/589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3/dataset/train/' <--> 'lakefs://is-alpaca/experiment-1/dataset/train/'...

╔════════╦════════╦════════════════════════════╗
║ SOURCE ║ CHANGE ║ PATH                       ║
╠════════╬════════╬════════════════════════════╣
║ local  ║ added  ║ not_alpaca/axolotl2.jpeg   ║
║ local  ║ added  ║ not_alpaca/axolotl3.png    ║
║ local  ║ added  ║ not_alpaca/axolotl4.jpeg   ║
╚════════╩════════╩════════════════════════════╝

is_alpaca$ lakectl local commit input -m "add images of axolotls to the training dataset"

Getting branch: experiment-1

diff 'local:///ml_models/is_alpaca/input' <--> 'lakefs://is-alpaca/589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3/dataset/train/'...
upload not_alpaca/axolotl3.png              ... done! [5.04KB in 679ms]
upload not_alpaca/axolotl2.jpeg             ... done! [38.31KB in 685ms]
upload not_alpaca/axolotl4.jpeg             ... done! [7.70KB in 718ms]

Sync Summary:

Downloaded: 0
Uploaded: 3
Removed: 0

Finished syncing changes. Perform commit on branch...
Commit for branch "experiment-1" completed.

ID: 0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6
Message: add images of axolotls to the training dataset
Timestamp: 2024-02-08 17:41:20 +0200 IST
Parents: 589f87704418c6bac80c5a6fc1b52c245af347b9ad1ea8d06597e4437fae4ca3

is_alpaca$ lakectl local init lakefs://is-alpaca/main/dataset/test/ testDataset 
Location added to /is_alpaca/.gitignore
Successfully linked local directory '/is_alpaca/testDataset' with remote 'lakefs://is-alpaca/main/dataset/test/'

is_alpaca$ lakectl local list                                                           
+-------------+-------------------------------------------------+------------------------------------------------------------------+
| DIRECTORY   | REMOTE URI                                      | SYNCED COMMIT                                                    |
+-------------+-------------------------------------------------+------------------------------------------------------------------+
| input       | lakefs://is-alpaca/main/dataset/train/          | 0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6 |
| testDataset | lakefs://is-alpaca/main/dataset/test/           | 0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6 |
+-------------+-------------------------------------------------+------------------------------------------------------------------+

is_alpaca$ lakectl local status testDataset 

diff 'local:///ml_models/is_alpaca/testDataset' <--> 'lakefs://is-alpaca/0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6/dataset/test/'...
diff 'lakefs://is-alpaca/0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6/dataset/test/' <--> 'lakefs://is-alpaca/main/dataset/test/'...

╔════════╦════════╦════════════════════════════════╗
║ SOURCE ║ CHANGE ║ PATH                           ║
╠════════╬════════╬════════════════════════════════╣
║ local  ║ added  ║ alpaca/alpaca (1).jpg          ║
║ local  ║ added  ║ alpaca/alpaca (10).jpg         ║
    .        .                  .
    .        .                  .
    .        .                  .
║ local  ║ added  ║ not_alpaca/not_alpaca (9).jpg  ║
╚════════╩════════╩════════════════════════════════╝

is_alpaca$ lakectl local commit testDataset -m "add is_alpaca test dataset to lakeFS" 

Getting branch: experiment-1

...

Finished syncing changes. Perform commit on branch...
Commit for branch "experiment-1" completed.

ID: c8be7f4f5c13dd2e489ae85e6f747230bfde8e50f9cd9b6af20b2baebfb576cf
Message: add is_alpaca test dataset to lakeFS
Timestamp: 2024-02-10 12:31:53 +0200 IST
Parents: 0b376f01b925a075851bbaffacf104a80de04a43ed7e56054bf54c42d2c8cce6