From 9fe8bb8cc3b2df74e27c691a5ef771065bc38d3f Mon Sep 17 00:00:00 2001 From: Phil Estival Date: Thu, 16 Jan 2025 12:11:01 +0100 Subject: [PATCH 01/11] org-sql.el: new variables and requirements for session support. * lisp/org-sql.el: requires sql.el for a connection to a session. Custom variables are declared in a new sub-group ob-babel-sql. SQL clients are configured by a preamble of commands given to the SQL shell. The echo of an SQL ANSI comment is appended to the source block of SQL commands for comint to detect when the commands terminate. --- lisp/ob-sql.el | 213 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) diff --git a/lisp/ob-sql.el b/lisp/ob-sql.el index 14ca6bc48..c149016cf 100644 --- a/lisp/ob-sql.el +++ b/lisp/ob-sql.el @@ -4,6 +4,7 @@ ;; Author: Eric Schulte ;; Maintainer: Daniel Kraus +;; Maintainer: Philippe Estival ;; Keywords: literate programming, reproducible research ;; URL: https://orgmode.org @@ -75,6 +76,33 @@ (org-assert-version) (require 'ob) +(require 'sql) + +(defvar org-babel-sql-session-start-time) +(defvar org-sql-session-preamble + (list + 'postgres "\\set ON_ERROR_STOP 1 +\\pset footer off +\\pset pager off +\\pset format unaligned" ) + "Command preamble to run upon shell start.") +(defvar org-sql-session-command-terminated nil) +(defvar org-sql-session--batch-terminate "---#" "To print at the end of a command batch.") +(defvar org-sql-batch-terminate + (list 'sqlite (format ".print %s\n" org-sql-session--batch-terminate) + 'postgres (format "\\echo %s\n" org-sql-session--batch-terminate)) + "Print the command batch termination as last command.") +(defvar org-sql-terminal-command-prefix + (list 'sqlite "\\." + 'postgres "\\\\") + "Identify a command for the SQL shell.") +(defvar org-sql-environment + (list 'postgres '(("PGPASSWORD" sql-password)))) +(defvar org-sql-session-clean-output nil + "Store the regexp used to clear output (prompt1|termination|prompt2).") +(defvar org-sql-session-start-time) +(defvar org-sql-session-command-terminated nil) +(defvar org-sql-session--batch-terminate "---#" "To print at the end of a command batch.") (declare-function org-table-import "org-table" (file arg)) (declare-function orgtbl-to-csv "org-table" (table params)) @@ -85,6 +113,24 @@ (defvar sql-connection-alist) (defvar org-babel-default-header-args:sql '()) +(defcustom org-sql-run-comint-p 'nil + "Run non-session SQL commands through comint if not nil." + :type '(boolean) + :group 'org-babel-sql + :safe t) + +(defcustom org-sql-timeout '5.0 + "Abort on timeout." + :type '(number) + :group 'org-babel-sql + :safe t) + +(defcustom org-sql-close-out-temp-buffer-p 'nil + "To automatically close sql-out-temp buffer." + :type '(boolean) + :group 'org-babel-sql + :safe t) + (defconst org-babel-header-args:sql '((engine . :any) (out-file . :any) @@ -433,6 +479,173 @@ argument mechanism." "Raise an error because Sql sessions aren't implemented." (error "SQL sessions not yet implemented")) +(defun org-babel-sql-session-connect (in-engine params session) + "Start the SQL client of IN-ENGINE if it has not. +PARAMS provides the sql connection parameters for a new or +existing SESSION. Clear the intermediate buffer from previous +output, and set the process filter. Return the comint process +buffer." + (let* ((buffer-name (format "%s" (if (string= session "none") "" + (format "[%s]" session)))) + (ob-sql-buffer (format "*SQL: %s*" buffer-name))) + + ;; initiate a new connection + (when (not (org-babel-comint-buffer-livep ob-sql-buffer)) + (save-window-excursion + (setq ob-sql-buffer ; start the client + (org-babel-sql-connect in-engine buffer-name params))) + (let ((sql-term-proc (get-buffer-process ob-sql-buffer))) + (unless sql-term-proc + (user-error (format "SQL %s didn't start" in-engine))) + + (with-current-buffer (get-buffer ob-sql-buffer) + ;; preamble commands + (let ((preamble (plist-get org-sql-session-preamble in-engine))) + (when preamble + (process-send-string ob-sql-buffer preamble) + (comint-send-input)))) + ;; let the preamble execution finish and be filtered + (sleep-for 0.1))) + + ;; set the redirection filter and return the SQL client buffer + (set-process-filter (get-buffer-process ob-sql-buffer) + #'org-sql-session-comint-output-filter) + (get-buffer ob-sql-buffer))) + +(defun org-babel-sql-connect (&optional engine sql-cnx params) + "Run ENGINE interpreter as an inferior process. +SQL-CNX is the client buffer. This is a variant from sql.el that prompt +parametrs for authentication only if there's a missing parameter. +Depending on the sql client the password should also be prompted." + + (setq sql-product(cond + ((assoc engine sql-product-alist) ; Product specified + engine) + (t sql-product))) ; or default to sql-engine + + (when (sql-get-product-feature sql-product :sqli-comint-func) + (let (;(buf (sql-find-sqli-buffer sql-product sql-connection)) ; unused yet + (sql-server (cdr (assoc :dbhost params))) + ;; (sql-port (cdr (assoc :port params))) ; todo + (sql-database (cdr (assoc :database params))) + (sql-user (cdr (assoc :dbuser params))) + (sql-password (cdr (assoc :dbpassword params))) + (prompt-regexp (sql-get-product-feature engine :prompt-regexp )) + (prompt-cont-regexp (sql-get-product-feature engine :prompt-cont-regexp)) + sqli-buffer + rpt) + ;; store the regexp used to clear output (prompt1|indicator|prompt2) + (setq org-sql-session-clean-output + (plist-put org-sql-session-clean-output engine + (concat "\\(" prompt-regexp "\\)" + "\\|\\(" org-sql-session--batch-terminate "\n\\)" + (when prompt-cont-regexp + (concat "\\|\\(" prompt-cont-regexp "\\)"))))) + ;; Get credentials. + ;; either all fields are provided + ;; or there's a specific case were no login is needed + ;; or trigger the prompt + (or (and sql-database sql-user sql-server) + (eq sql-product 'sqlite) ;; sqlite allows in-memory db, w/o login + (apply #'sql-get-login + (sql-get-product-feature engine :sqli-login))) + ;; depending on client, password is forcefully prompted + + ;; The password wallet returns a function + ;; which supplies the password. (untested) + (when (functionp sql-password) + (setq sql-password (funcall sql-password))) + + ;; Erase previous sql-buffer. + ;; Will look for it's prompt to indicate session readyness. + (let ((previous-session + (get-buffer (format "*SQL: %s*" sql-cnx)))) + (when previous-session + (with-current-buffer + previous-session (erase-buffer))) + + (setq sqli-buffer + (let ((process-environment (copy-sequence process-environment)) + (variables (plist-get org-sql-environment engine))) + (mapc (lambda (elem) ; environment variables, evaluated here + (setenv (car elem) (eval (cadr elem)))) + variables) + (funcall (sql-get-product-feature engine :sqli-comint-func) + engine + (sql-get-product-feature engine :sqli-options) + (format "SQL: %s" sql-cnx)))) + (setq sql-buffer (buffer-name sqli-buffer)) + + (setq rpt (sql-make-progress-reporter nil "Login")) + (with-current-buffer sql-buffer + (let ((proc (get-buffer-process sqli-buffer)) + (secs org-sql-timeout) + (step 0.2)) + (while (and proc + (memq (process-status proc) '(open run)) + (or (accept-process-output proc step) + (<= 0.0 (setq secs (- secs step)))) + (progn (goto-char (point-max)) + (not (re-search-backward + prompt-regexp 0 t)))) + (sql-progress-reporter-update rpt))) + + ;; no prompt, connexion failed (and process is terminated) + (goto-char (point-max)) + (unless (re-search-backward prompt-regexp 0 t) + (user-error "Connection failed"))) ;is this a _user_ error? + ;;(run-hooks 'sql-login-hook) ; don't + ) + (sql-progress-reporter-done rpt) + (get-buffer sqli-buffer)))) + +(defun org-sql-session-format-query (str in-engine) + "Process then send the command STR to the SQL process. +Provide IN-ENGINE to retrieve product features. +Carefully separate client commands from SQL commands +Concatenate SQL commands as one line is one way to stop on error. +Otherwise the entire batch will be emitted no matter what. +Finnally add the termination command." + (concat + (let ((commands (split-string str "\n")) + (terminal-command + (concat "^\s*" + (plist-get org-sql-terminal-command-prefix in-engine)))) + (mapconcat + (lambda(s) + (when (not + (string-match "\\(^[\s\t]*--.*$\\)\\|\\(^[\s\t]*$\\)" s)) + (concat (replace-regexp-in-string + "[\t]" "" ; filter tabs + (replace-regexp-in-string "--.*" "" s)) ;; remove comments. + ;; Note: additional filtering is required for Vertica C-style comments. + (when (string-match terminal-command s) "\n")))) + commands " " )) + ";\n" + (plist-get org-sql-batch-terminate in-engine) + "\n" )) + +(defun org-sql-session-comint-output-filter (_proc string) + "Process output STRING of PROC gets redirected to a temporary buffer. +It is called several times consecutively as the shell outputs and flush +its message buffer" + + ;; Inserting a result in the sql process buffer (to read it as a + ;; regular prompt log) inserts it to the terminal, and as a result the + ;; ouput would get passed as input onto the next command line; See + ;; `comint-redirect-setup' to possibly fix that, + ;; (with-current-buffer (process-buffer proc) (insert output)) + + (when (or (string-match org-sql-session--batch-terminate string) + (> (time-to-seconds + (time-subtract (current-time) + org-sql-session-start-time)) + org-sql-timeout)) + (setq org-sql-session-command-terminated t)) + + (with-current-buffer (get-buffer-create "*ob-sql-result*") + (insert string))) + (provide 'ob-sql) ;;; ob-sql.el ends here -- 2.39.5