XTDB 2.x Early Access

;; deps.edn
{:mvn/repos {"ossrh-snapshots" {:url "https://s01.oss.sonatype.org/content/repositories/snapshots"}}
 :deps {org.clojure/clojure {:mvn/version "1.11.1"}
        com.xtdb.labs/xtdb-api {:mvn/version "2.0.0-SNAPSHOT"}
        com.xtdb.labs/xtdb-core {:mvn/version "2.0.0-SNAPSHOT"}
        com.xtdb.labs/xtdb-http-server {:mvn/version "2.0.0-SNAPSHOT"}
        com.xtdb.labs/xtdb-pgwire-server {:mvn/version "2.0.0-SNAPSHOT"}}
 :aliases
 {:xtdb {:jvm-opts
          ;; needed on JDK16+
         ["--add-opens=java.base/java.nio=ALL-UNNAMED"
          "-Dio.netty.tryReflectionSetAccessible=true"]}
  ;; If you want to connect using nREPL
  :nrepl
  {:main-opts ["-m" "nrepl.cmdline"]
   :extra-deps {nrepl/nrepl {:mvn/version "1.0.0"}}}}}

;; Once you have a REPL (started with clj -A:xtdb -M:nrepl if you’re on JDK 16+), you can create an in-memory XTDB node with:
(require '[xtdb.node :as xt.node]
         '[xtdb.api :as xt])

(def my-node (xt.node/start-node {:xtdb/server {:port 3001}
                                  :xtdb/pgwire {:port 5432}}))

;; Confirm this API call returns successfully
(xt/status my-node)

# Use this one-line shell command to start a Clojure REPL + XTDB server:
clojure -Sdeps '{:mvn/repos {"ossrh-snapshots" {:url "https://s01.oss.sonatype.org/content/repositories/snapshots"}}
                 :deps {org.clojure/clojure {:mvn/version "1.11.1"}
                        com.bhauman/rebel-readline {:mvn/version "0.1.4"}
                        com.xtdb.labs/xtdb-api {:mvn/version "2.0.0-SNAPSHOT"}
                        com.xtdb.labs/xtdb-core {:mvn/version "2.0.0-SNAPSHOT"}
                        com.xtdb.labs/xtdb-http-server {:mvn/version "2.0.0-SNAPSHOT"}
                        com.xtdb.labs/xtdb-pgwire-server {:mvn/version "2.0.0-SNAPSHOT"}}
                 ;; needed on JDK16+
                 :aliases {:xtdb {:jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED"
                             "-Dio.netty.tryReflectionSetAccessible=true"]}}}' \
        -M -e '(do (require (quote [xtdb.node :as xt.node])
                            (quote [xtdb.api :as xt])
                            (quote [rebel-readline.main]))
                   (def my-node (xt.node/start-node {:xtdb/server {:port 3001}
                                  :xtdb/pgwire {:port 5432}}))
                   (rebel-readline.main/-main))' -r
        

# Boot the 2.x Early Access Docker image in seconds.
docker pull ghcr.io/xtdb/xtdb-ea 
docker run -p 5432:5432 -p 3001:3000 ghcr.io/xtdb/xtdb-ea

;; deps.edn
{:mvn/repos {"ossrh-snapshots" {:url "https://s01.oss.sonatype.org/content/repositories/snapshots"}}
 :deps {org.clojure/clojure {:mvn/version "1.11.1"}
        com.xtdb.labs/xtdb-api {:mvn/version "2.0.0-SNAPSHOT"}
        com.xtdb.labs/xtdb-http-client-clj {:mvn/version "2.0.0-SNAPSHOT"}}}

;; Once you have a repl started ...
(require '[xtdb.client :as xt.client]
         '[xtdb.api :as xt])

(def my-node (xt.client/start-client "http://localhost:3001"))

(xt/status my-node)

# Use this one-line shell command to start a Clojure REPL
# It connects to your XTDB container automatically:
clojure -Sdeps '{:mvn/repos {"ossrh-snapshots" {:url "https://s01.oss.sonatype.org/content/repositories/snapshots"}}
                 :deps {org.clojure/clojure {:mvn/version "1.11.1"}
                        com.bhauman/rebel-readline {:mvn/version "0.1.4"}
                        com.xtdb.labs/xtdb-api {:mvn/version "2.0.0-SNAPSHOT"}
                        com.xtdb.labs/xtdb-http-client-clj {:mvn/version "2.0.0-SNAPSHOT"}}}' \
        -M -e '(do (require (quote [xtdb.client :as xt.client])
                            (quote [xtdb.api :as xt])
                            (quote [rebel-readline.main]))
                   (def my-node (xt.client/start-client "http://localhost:3001"))
                   (rebel-readline.main/-main))' -r

# Use any Postgres-compatible client.
psql -h localhost

(xt/submit-tx my-node [[:put :posts {:xt/id 1234
                                     :user-id 5678
                                     :text "hello world!"}]])

(xt/q my-node '{:find [text]
                :where [($ :posts [text])]})

INSERT INTO posts (xt$id, user_id, text)
  VALUES (1234, 5678, 'Hello World!');

SELECT posts.text FROM posts;

;; XTDB has always handled ~arbitrary data well - just smash your maps in!
(xt/submit-tx my-node [[:put :people {:xt/id 5678
                                      :name "Sarah"
                                      :friends [{:user "Dan"}
                                                {:user "Kath"}]}]])

;; Datalog now supports first-class nested lookups too:
(xt/q my-node '{:find [friend]
                :where [($ :people [friends])
                        [(nth friends 1) friend]]})

-- Nested data is natural!
-- XTDB SQL makes working with Arrays and Objects easy,
-- these are not opaque or limited JSONB types,
-- instead they are seamlessly integrated with SQL:
INSERT INTO people (xt$id, name, friends)
  VALUES (5678, 'Sarah',
          [{'user': 'Dan'},
           {'user': 'Kath'}]);

SELECT people.friends[2] FROM people;

;; Datalog uses as-of-now defaults
;; To view the entire history of a record:
(xt/q my-node '{:find [person
                       valid-from valid-to
                       system-from system-to]
                :where [($ :people [{:xt/id 5678
                                     :xt/* person
                                     :xt/valid-from valid-from
                                     :xt/valid-to valid-to
                                     :xt/system-from system-from
                                     :xt/system-to system-to}]
                                   {:for-valid-time :all-time
                                    :for-system-time :all-time})]})

-- XTDB SQL defaults to the SQL:2011 standard
-- using 'SET VALID_TIME_DEFAULTS TO ISO_STANDARD;'
-- this means that all of time visible by default
-- but it also offers alternative defaults for a more intuitive experience:
SET VALID_TIME_DEFAULTS TO AS_OF_NOW;

SELECT *
FROM people
         FOR ALL SYSTEM_TIME
         FOR ALL VALID_TIME
         AS x (xt$id,
               name,
               friends,
               xt$valid_from,
               xt$valid_to,
               xt$system_from,
               xt$system_to);

(xt/submit-tx my-node [[:put :posts
                             {:xt/id 9012 ;; original post
                              :user-id 5678
                              :text "Happy 2019!"}
                              {:for-valid-time
                               [:in #inst "2019-01-01" nil]}]
                       [:put :posts
                             {:xt/id 3456 ;; later post
                              :user-id 5678
                              :text "(New) Happy 2024!"}
                              {:for-valid-time
                               [:in #inst "2024-01-01" nil]}]
                       [:put :posts
                             {:xt/id 9012 ;; original post again
                              :user-id 5678
                              :text "Edited-2019-post: Happy 2025!"}
                              {:for-valid-time
                               [:in #inst "2025-01-01" nil]}]])

(xt/q my-node '{:find [text]
                :where [($ :posts [text]
                                  {:for-valid-time
                                   [:at #inst "2025-01-02"]})]})

(xt/q my-node '{:find [text]
                :where [($ :posts [text])]})

;; what other (versions of) posts were valid during the same time range as the 2019 post?
(xt/q my-node '{:find [text]
                :where [($ :posts [{:xt/id 3456
                                    :xt/valid-time vt}]
                                  {:for-valid-time :all-time})
                        ($ :posts [{:xt/id other-id
                                    :xt/valid-time other-vt} text]
                                   {:for-valid-time :all-time})
                        [(<> other-id 3456)]
                        [(overlaps? vt other-vt)]]})

INSERT INTO posts (xt$id, user_id, text, xt$valid_from)
  VALUES (9012, 5678, 'Happy 2019!', DATE '2019-01-01'),
         (3456, 5678, '(New) Happy 2024!', DATE '2024-01-01'),
         (9012, 5678, 'Edited-2019-post: Happy 2025!', DATE '2025-01-01');

SELECT posts.text FROM posts
  FOR VALID_TIME AS OF DATE '2025-01-02';

SELECT posts.text FROM posts;

SELECT P2.text
FROM posts FOR ALL VALID_TIME AS P1,
     posts FOR ALL VALID_TIME AS P2
WHERE P1.xt$id = 3456
  AND P2.xt$id <> 3456
  AND P1.VALID_TIME OVERLAPS P2.VALID_TIME;

SELECT more_posts.text
  FROM ARROW_TABLE('https://xtdb.com/more_posts.arrow')
  AS more_posts;

;; Coming soon!

;; we are going to rename :evict to :erase in the near future
(xt/submit-tx my-node [[:evict :people 5678]])

ERASE FROM people WHERE people.xt$id=5678;

What next?

Why not Learn Datalog Today! Or generate your own playground starter project using our Clojure templates!

New for 1.x users

The 2.x design expands on the essential qualities of 1.x and provides significant new capabilities:

• “Cross time” queries, e.g. temporal joins and temporal range scans - where 1.x is optimized only for point-in-time queries
• Massive and affordable scale via a separated storage and compute architecture, using object storage for handling large chunks of Apache Arrow data on demand
• Extensive relational algebra optimizations (e.g. subquery decorrelation)
• First-class SQL experience alongside Datalog with an entirely new SQL parser and planner (no longer using Apache Calcite)
• Organize records into tables for performance (query planning, partitioning etc.), introspection, and seamless SQL interoperability
• Datalog enriched with SQL-derived functionality, with a SQL-based core library, “Bag” / “multi-set” semantics, and Three-Valued Logic (3VL) - supersedes 1.x Clojure classpath interop

The roadmap towards General Availability

This roadmap is an expression of intent and is subject to change.

• Additional storage backends: a full suite of cloud storage adapters optimized for various Kafka-like and S3-like managed services
• Monitoring: database health, OpenTelemetry
• Deployment: zero-configuration Kubernetes setup for managed platforms (i.e. EKS/AKS/GKS)
• Tiered caching: block management, clustering
• Content indexing: light & adaptive indexes
• Benchmarking: fine-tuning against AuctionMark, TPC-H and beyond
• Hardening: transaction processing, execution engine compilation, complex Arrow usage
• Join planning: additional statistics (e.g. HyperLogLog, histograms)
• Language drivers: e.g. Java & Kotlin
• SQL drivers: evolved pgwire-server support (currently all data is returned as JSON), dynamic Flight SQL
• API Completeness - for example:
  • INFORMATION_SCHEMA: users should be able introspect all data and aspects of the database as built-in virtual tables through the primary query APIs. e.g. an initial version of the `xt$txs` table is already implemented (Change Data Feed functionality to follow)
  • User-definable transaction metadata: improved provenance, "Event sourcing without the hassle"
  • Database VIEWs: gradual schema, writeable views, nested views, materialized views
  • Batch writes: concurrent & atomic bulk loading of data
  • Pull-like API: nested data retrieval that is more ergonomic than scalar subqueries
  • Ongoing SQL enhancements: e.g. implement additional ISO specification functions and operators