HTTPX Client Example

In [ ]:

# Use any available huggingface model
MODEL = "/models/meta-llama/Llama-3.2-1B-Instruct/"
PORT = 7621

# Use any available huggingface model MODEL = "/models/meta-llama/Llama-3.2-1B-Instruct/" PORT = 7621

In [ ]:

# Launch vLLM with output protection
!python -m stainedglass_output_protection.vllm.entrypoint --model {MODEL} --port {PORT}

# Launch vLLM with output protection !python -m stainedglass_output_protection.vllm.entrypoint --model {MODEL} --port {PORT}

In [2]:

import httpx

import httpx

In [8]:

client = httpx.Client(base_url=f"http://localhost:{PORT}")

client = httpx.Client(base_url=f"http://localhost:{PORT}")

In [9]:

response = client.get("/v1/models")
response.json()

response = client.get("/v1/models") response.json()

Out[9]:

{'object': 'list',
 'data': [{'id': '/models/meta-llama/Llama-3.2-1B-Instruct/',
   'object': 'model',
   'created': 1740171409,
   'owned_by': 'vllm',
   'root': '/models/meta-llama/Llama-3.2-1B-Instruct/',
   'parent': None,
   'max_model_len': 131072,
   'permission': [{'id': 'modelperm-182b6dbcfb364d1a8c4767fa3c6ab1b3',
     'object': 'model_permission',
     'created': 1740171409,
     'allow_create_engine': False,
     'allow_sampling': True,
     'allow_logprobs': True,
     'allow_search_indices': False,
     'allow_view': True,
     'allow_fine_tuning': False,
     'organization': '*',
     'group': None,
     'is_blocking': False}]}]}

In [ ]:

import base64
import json

import transformers
from cryptography.hazmat.primitives.asymmetric import x25519
from stainedglass_output_protection import encryption

client_private_key, client_public_key = (
    encryption.generate_ephemeral_keypair()
)  # pragma: allowlist secret

tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)


prompt = tokenizer.apply_chat_template(
    [
        {"role": "system", "content": "You are Shakespeare."},
        {
            "role": "user",
            "content": "Please write me a short sonnet about frogs who sit on logs.",
        },
    ],
    tokenize=False,
)

print(prompt)
print(
    "===================================================================================================================="
)

response = client.post(
    "/v1/completions",
    json={"model": MODEL, "prompt": prompt, "max_tokens": 256},
    headers={
        "x-public-key": base64.b64encode(
            client_public_key.public_bytes_raw()
        ).decode("utf-8")
    },
)

print(json.dumps(response.json(), indent=2))
print(
    "===================================================================================================================="
)
print(response.headers)
print(
    "===================================================================================================================="
)

server_public_key = x25519.X25519PublicKey.from_public_bytes(
    base64.b64decode(response.headers["x-server-public-key"])
)  # pragma: allowlist secret

shared_key = encryption.derive_shared_aes_key(
    client_private_key, server_public_key
)  # pragma: allowlist secret

ciphertext = base64.b64decode(response.json()["choices"][0]["text"])
print(encryption.decrypt(ciphertext, shared_aes_key=shared_key).decode("utf-8"))

import base64 import json import transformers from cryptography.hazmat.primitives.asymmetric import x25519 from stainedglass_output_protection import encryption client_private_key, client_public_key = ( encryption.generate_ephemeral_keypair() ) # pragma: allowlist secret tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL) prompt = tokenizer.apply_chat_template( [ {"role": "system", "content": "You are Shakespeare."}, { "role": "user", "content": "Please write me a short sonnet about frogs who sit on logs.", }, ], tokenize=False, ) print(prompt) print( "====================================================================================================================" ) response = client.post( "/v1/completions", json={"model": MODEL, "prompt": prompt, "max_tokens": 256}, headers={ "x-public-key": base64.b64encode( client_public_key.public_bytes_raw() ).decode("utf-8") }, ) print(json.dumps(response.json(), indent=2)) print( "====================================================================================================================" ) print(response.headers) print( "====================================================================================================================" ) server_public_key = x25519.X25519PublicKey.from_public_bytes( base64.b64decode(response.headers["x-server-public-key"]) ) # pragma: allowlist secret shared_key = encryption.derive_shared_aes_key( client_private_key, server_public_key ) # pragma: allowlist secret ciphertext = base64.b64decode(response.json()["choices"][0]["text"]) print(encryption.decrypt(ciphertext, shared_aes_key=shared_key).decode("utf-8"))

<|begin_of_text|><|start_header_id|>system<|end_header_id|>

Cutting Knowledge Date: December 2023
Today Date: 21 Feb 2025

You are Shakespeare.<|eot_id|><|start_header_id|>user<|end_header_id|>

Please write me a short sonnet about frogs who sit on logs.<|eot_id|>
====================================================================================================================
{
  "id": "cmpl-5cff307feb824339810fa6e79d94d8da",
  "object": "text_completion",
  "created": 1740171410,
  "model": "/models/meta-llama/Llama-3.2-1B-Instruct/",
  "choices": [
    {
      "index": 0,
      "text": "h+Br+G9MYpoFEovGpBQy43ewHYmXNj04vCnb0rPEEVBUb8qj8Kpgwj6M3HbqD1EY/Y4XwKMWgCVwDp5B0nImGCFVgPoLUOw/zS8CgZn1b8XGV8aIq8gDW2E6TfQZyfy+2antrgQaHhJhihymKQvdWEiBTdDpYjRAwQxoylfTB06awFLDx1r+ZwmBai2Yxa+9ZnBohGMHqAsEGzk1jo5v/+s6La4WCtjnfzPmHbjk91j8iNNLzcWdvYt1gSDcvnUnB6wS9Z7Oj6pGIkCImxmvsPzEbqL577GUU5HJDiOGTcHOEndzaQpeiI1Q1qjL0WmHxbOQeBVIjo25sUqPpaOZxlrwfSV1hI4kDNsmpf3hZs01Vy+ii3Z10et3vdxRVBuoHEGpqSKYw87VtVCO3KIS+WOdQ2vfcP0XFrd4NlRWSKZPM+RhFCB+ugFY4qcahYcnu/eIMsj2TeCf9pyZfNNbwix41aX9uStFA+OoJG6I+/c6TQrKutUB8uTdLvrpe8zU4hO9BegTZBTWTTQjlGjXZ99JVa8sQE3cLKdiMSLOfQQ7jwMqywQuE7jlWNnfvzVGTglJeMFFusFZTGZPFvFWZZMx0M4qQKMdJuVabVIrj8Z6WDtY1VxfY4JUpCqPrHq+5GiYkbC21rXCZcOU3G5NngKNOlR+16ZxGEBXMAcXm+T5QoJ2tQToiLfrSmapeSdj3kk1d0o0UBR2tApXCmMv9LlESM8aQ7Bm1stdLP2ffGxUmmWpnk0mVTiBtrGFOvHvUG9Vs7JYCbR+Koq6UcrqbBUQZrx7JDcIKXiWNHjJo8sJCJR2qpdjkH1Nu8xoMi+b+rlyGxe0O7a0FizTkTz8+l5AhYHbapyegxe/ldwZ859nr8PMfS2bP9MjazV3rDH9y9dFzJxFdbCnBuEhA7rb4cDVo9/n1exfsy7BUKhPeG0D3YqBdWdE27+e3L0=",
      "logprobs": null,
      "finish_reason": "stop",
      "stop_reason": null,
      "prompt_logprobs": null
    }
  ]
}
====================================================================================================================
Headers({'date': 'Fri, 21 Feb 2025 20:56:49 GMT', 'server': 'uvicorn', 'content-length': '1284', 'content-type': 'application/json', 'x-server-public-key': 'LR6nDESMnEO0IFPDOyQWThtMhBsgwwaOLGbEqV8udkA='})
====================================================================================================================
<|start_header_id|>assistant<|end_header_id|>

In twilight's hush, where shadows dance and play,
Amidst the logs that slumber still in night,
A chorus chorus of frogs do laugh and sway,
Their melancholy serenade takes flight.

Their evening song, a gentle, soothing hush,
That wraps the world in quiet, leafy peace,
As if to hide their troubles, rush and rush,
But in their deep bark, a secret release.

Their emerald eyes, like jewels dark and bright,
Do sparkle with a glimmer of wild delight,
Their powerful limbs, that lift and hold the light,
Do tremble with a joy, so pure and bright.

In woods, their soft voice, a magic spell doth cast,
That beckons all to join their peaceful_PAD

In [ ]: