Adding upstream version 3.0.16.

Signed-off-by: Daniel Baumann <daniel@debian.org>

.github/workflows/test.yaml

@@ -0,0 +1,38 @@
+name: test
+
+on:
+  push: # any branch
+  pull_request:
+    branches: [master]
+
+jobs:
+  test-ubuntu:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          sudo apt remove python3-pip
+          python -m pip install --upgrade pip
+          python -m pip install . black isort mypy pytest readme_renderer
+          pip list
+      - name: Type Checker
+        run: |
+          mypy ptpython
+          isort -c --profile black ptpython examples setup.py
+          black --check ptpython examples setup.py
+      - name: Run Tests
+        run: |
+          ./tests/run_tests.py
+      - name: Validate README.md
+        # Ensure that the README renders correctly (required for uploading to PyPI).
+        run: |
+          python -m readme_renderer README.rst > /dev/null

.gitignore

@@ -0,0 +1,54 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/

CHANGELOG

@@ -0,0 +1,587 @@
+CHANGELOG
+=========
+
+3.0.16: 2020-02-11
+------------------
+
+(Commit 7f619e was missing in previous release.)
+
+Fixes:
+- Several fixes to the completion code:
+  * Give dictionary completions priority over path completions.
+  * Always call non-fuzzy completer after fuzzy completer to prevent that some
+    completions were missed out if the fuzzy completer doesn't find them.
+
+
+3.0.15: 2020-02-11
+------------------
+
+New features:
+- When pressing control-w, only delete characters until a punctuation.
+
+Fixes:
+- Fix `AttributeError` during retrieval of signatures with type annotations.
+
+
+3.0.14: 2020-02-10
+------------------
+
+New features:
+- Display of signature and completion drop down together.
+- If `DictionaryCompleter` is enabled, also retrieve signatures when Jedi
+  fails, using the same logic.
+- List function parameters first and private attributes at the end in the
+  completion menu.
+- Cleanup of the completion code.
+
+Fixes:
+- Handle exceptions raised when `repr()` is called.
+- Fix leakage of `exc_info` from eval to exec call.
+- Fix handling of `KeyboardInterrupt` in REPL during evaluation of `__repr__`.
+- Fix style for signature toolbar.
+- Hide signature when sidebar is visible.
+
+
+3.0.13: 2020-01-26
+------------------
+
+New features:
+- Added 'print all' option to pager.
+- Improve handling of indented code:
+  * Allow multiline input to be indented as a whole (we will unindent before
+    executing).
+  * Correctly visualize tabs (instead of ^I, which happens when pasted in
+    bracketed paste).
+
+Fixes:
+- Fix line ending bug in pager.
+
+
+3.0.12: 2020-01-24
+------------------
+
+New features:
+- Expose a `get_ptpython` function in the global namespace, to get programmatic
+  access to the REPL.
+- Expose `embed()` at the top level of the package. Make it possible to do
+  `from ptpython import embed`.
+
+Fixes:
+- Properly handle exceptions when trying to access `__pt_repr__`.
+- Properly handle `SystemExit`.
+
+
+3.0.11: 2020-01-20
+------------------
+
+New features:
+- Add support for top-level await.
+- Refactoring of event loop usage:
+
+  * The ptpython input UI will now run in a separate thread. This makes it
+    possible to properly embed ptpython in an asyncio application, without
+    having to deal with nested event loops (which asyncio does not support).
+
+  * The "eval" part doesn't anymore take place within a ptpython coroutine, so
+    it can spawn its own loop if needed. This also fixes `asyncio.run()` usage
+    in the REPL, which was broken before.
+
+- Added syntax highlighting and autocompletion for !-style system commands.
+
+Fixes:
+- Remove unexpected additional line after output.
+- Fix system prompt. Accept !-style inputs again.
+- Don't execute PYTHONSTARTUP when -i flag was given.
+
+
+3.0.10: 2020-01-13
+------------------
+
+Fixes:
+- Do dictionary completion on Sequence and Mapping objects (from
+  collections.abc). Note that dictionary completion is still turned off by
+  default.
+
+
+3.0.9: 2020-01-10
+-----------------
+
+New features:
+- Allow replacing `PythonInput.completer` at runtime (useful for tools build on
+  top of ptpython).
+- Show REPL title in pager.
+
+
+3.0.8: 2020-01-05
+-----------------
+
+New features:
+- Optional output formatting using Black.
+- Optional pager for displaying outputs that don't fit on the screen.
+- Added --light-bg and --dark-bg flags to automatically optimize the brightness
+  of the colors according to the terminal background.
+- Addd `PTPYTHON_CONFIG_HOME` for explicitely setting the config directory.
+- Show completion suffixes (like '(' for functions).
+
+Fixes:
+- Fix dictionary completion on Pandas objects.
+- Stop using deprecated Jedi functions.
+
+
+3.0.7: 2020-09-25
+-----------------
+
+New features:
+- Option to show/hide private attributes during a completion
+- Added `insert_blank_line_after_input` option similar to
+  `insert_blank_line_after_output`.
+
+Fixes:
+- Fixed some formatting issues of `__pt_repr__`.
+- Abbreviate completion meta information for dictionary completer if needed.
+
+
+3.0.6: 2020-09-23
+-----------------
+
+New features:
+- (Experimental) support for `__pt_repr__` methods. If objects implement this
+  method, this will be used to print the result in the REPL instead of the
+  normal `__repr__`.
+- Added py.typed file, to enable type checking for applications that are
+  embedding ptpython.
+
+
+3.0.5: 2020-08-10
+-----------------
+
+Fixes:
+- Handle bug in dictionary completion when numeric keys are used.
+
+
+3.0.4: 2020-08-10
+-----------------
+
+New features:
+- Allow leading whitespace before single line expressions.
+- Show full syntax error in validator.
+- Added `vi_start_in_navigation_mode` and `vi_keep_last_used_mode` options.
+
+Fixes:
+- Improved dictionary completion: handle keys that contain spaces and don't
+  recognize numbers as variable names.
+- Fix in exit confirmation.
+
+
+3.0.3: 2020-07-10
+-----------------
+
+Fixes:
+- Sort attribute names for `DictionaryCompleter` and move underscored
+  attributes to the end.
+- Handle unhandled exceptions in `get_compiler_flags`.
+- Improved `run_async` code.
+- Fix --version parameter.
+
+
+3.0.2: 2020-04-14
+-----------------
+
+New features:
+- Improved custom dictionary completion:
+    * Also complete list indexes.
+    * Also complete attributes after doing a dictionary lookup.
+    * Also complete iterators in a for-loop.
+- Added a 'title' option, so that applications embedding ptpython can set a
+  title in the status bar.
+
+
+3.0.1: 2020-02-24
+-----------------
+
+- Fix backwards-compatibility of the `run_config` function. (used by
+  django-extensions).
+- Fix input mode in status bar for block selection.
+
+
+3.0.0: 2020-01-29
+-----------------
+
+Upgrade to prompt_toolkit 3.0.
+Requires at least Python 3.6.
+
+New features:
+- Uses XDG base directory specification.
+
+
+2.0.5: 2019-10-09
+-----------------
+
+New features:
+- Added dictionary completer (off by default).
+- Added fuzzy completion (off by default).
+- Highlight keywords in completion dropdown menu.
+- Enable universal wheels.
+
+Fixes:
+- Fixed embedding repl as asyncio coroutine.
+- Fixed patching stdout in embedded repl.
+- Fixed ResourceWarning in setup.py.
+
+
+2.0.4: 2018-10-30
+-----------------
+
+- Fixed ptipython.
+- Fixed config: setting of color depth.
+- Fixed auto-suggest key bindings.
+- Fixed Control-D key binding for exiting REPL when (confirm_exit=False).
+- Correctly focus/unfocus sidebar.
+- Fixed open_in_editor and suspend key bindings.
+
+
+2.0.3: 2018-10-12
+-----------------
+
+- Allow changing the min/max brightness.
+- Some changes for compatibility with the latest prompt_toolkit.
+
+2.0.2: 2018-09-30
+-----------------
+
+Fixes:
+- Don't crash the history browser when there was no history.
+- Set last exception in the sys module, when an exception was raised.
+- Require prompt_toolkit 2.0.5.
+
+
+2.0.1: 2018-09-30
+-----------------
+
+Upgrade to prompt_toolkit 2.0.x.
+
+
+0.36: 2016-10-16
+----------------
+
+New features:
+- Support for editing in Vi block mode. (Only enabled for
+  prompt_toolkit>=1.0.8.)
+
+Fixes:
+- Handle two Jedi crashes. (GitHub ptpython issues #136 and #91.)
+
+0.35: 2016-07-19
+----------------
+
+Fixes:
+- Fix in completer. Don't hang when pasting a long string with many
+  backslashes.
+- Fix Python2 bug: crash when filenames contain non-ascii characters.
+- Added `pt[i]pythonX` and `pt[i]pythonX.X` commands.
+- Compatibility with IPython 5.0.
+
+
+0.34: 2016-05-06
+---------------
+
+Bugfix in ptipython: reset input buffer before every read in run().
+
+
+0.33: 2016-05-05
+---------------
+
+Upgrade to prompt_toolkit 1.0.0
+
+Improvements:
+- Unindent after typing 'pass'.
+- Make it configurable whether or not a blank line has to be inserted after the output.
+
+
+0.32: 2016-03-29
+---------------
+
+Fixes:
+- Fixed bug when PYTHONSTARTUP was not found.
+- Support $PYTHONSTARTUP for ptipython.
+
+
+0.31: 2016-03-14
+---------------
+
+Upgrade to prompt_toolkit 0.60
+
+
+0.30: 2016-02-27
+---------------
+
+Upgrade to prompt_toolkit 0.59
+
+
+0.29: 2016-02-24
+----------------
+
+Upgrade to prompt_toolkit 0.58
+
+New features:
+- Improved mouse support
+
+
+0.28: 2016-01-04
+----------------
+
+Upgrade to prompt_toolkit 0.57
+
+
+0.27: 2016-01-03
+----------------
+
+Upgrade to prompt_toolkit 0.56
+
+
+0.26: 2016-01-03
+----------------
+
+Upgrade to prompt_toolkit 0.55
+
+Fixes:
+- Handle several bugs in Jedi.
+- Correctly handle sys.argv when pt(i)python is started with --interactive.
+- Support for 24bit true color.
+- Take compiler flags into account for ptipython.
+
+
+0.25: 2015-10-29
+----------------
+
+Upgrade to prompt_toolkit 0.54
+
+Fixes:
+- Consider input multiline when there's a colon at the end of the line.
+- Handle bug in Jedi.
+- Enable search bindings in history browser.
+
+
+0.24: 2015-09-24
+----------------
+
+Upgrade to prompt_toolkit 0.52
+
+
+0.23: 2015-09-24
+----------------
+
+Upgrade to prompt_toolkit 0.51
+
+New features:
+- Mouse support
+- Fish style auto suggestion.
+- Optionally disabling of line wraps.
+- Use Python3Lexer for Python 3.
+
+
+0.22: 2015-09-06
+----------------
+
+Upgrade to prompt_toolkit 0.50
+
+Fixes:
+- Correctly accept file parameter in the print function of
+  asyncssh_repl.ReplSSHServerSession.
+- Create config directory if it doesn't exist yet (For IPython entry point.)
+
+New features:
+- Implementation of history-selection: a tool to select lines from the history.
+- Make exit message configurable.
+- Improve start-up time: Lazy load completer grammar and lazy-import Jedi.
+- Make multi-column the default completion visualisation.
+- Implementation of a custom prompts. In_tokens and out_tokens can be
+  customized.
+- Made an option to show/hide highlighting for matching parenthesis.
+- Some styling improvements.
+
+
+0.21: 2015-08-08
+---------------
+
+Upgrade to prompt_toolkit 0.46
+
+Fixes:
+- Correctly add current directory to sys.path.
+- Only show search highlighting when the search is the current input buffer.
+- Styling fix.
+
+
+0.20: 2015-07-30
+---------------
+
+Upgrade to prompt_toolkit 0.45
+
+
+0.19: 2015-07-30
+---------------
+
+Upgrade to prompt_toolkit 0.44
+
+New features:
+- Added --interactive option for ptipython.
+- A few style improvements.
+
+
+0.18: 2015-07-15
+---------------
+
+Fixes:
+- Python 2.6 compatibility.
+
+
+0.17: 2015-07-15
+---------------
+
+Upgrade to prompt_toolkit 0.43
+
+New features:
+- Integration with Tk eventloop. (This makes turtle and other Tk stuff work
+  again from the REPL.)
+- Multi column completion visualisation.
+
+
+0.16: 2015-06-25
+---------------
+
+Upgrade to prompt_toolkit 0.42
+
+Fixes:
+- Workaround for Jedi bug. (Signatures of functions with keyword-only arguments.)
+- Correctly show traceback on Python 3.
+- Better styling of the options sidebar.
+
+New features:
+- Exit REPL when input starts with Control-Z.
+- Set terminal title.
+- Display help text in options sidebar.
+- Better colorscheme for Windows.
+
+
+0.15: 2015-06-20
+---------------
+
+Upgrade to prompt_toolkit 0.41
+
+Fixes:
+- Correct tokens for IPython prompt.
+- Syntax fix in asyncssh_repl.
+
+
+0.14: 2015-06-16
+---------------
+
+Fix:
+- Correct dependency for prompt_toolkit.
+
+0.13: 2015-06-15
+---------------
+
+New features:
+- Upgrade to prompt_toolkit 0.40
+- Options sidebar.
+- Custom color schemes.
+- Syntax highlighting of the output.
+- Input validation can now be turned off.
+- Determine compiler flags dynamically. (Fixes importing unicode_literals).
+- Exit confirmation and dialog.
+- Autocompletion of IPython %cat command.
+- Correctly render tracebacks on Windows.
+
+0.12: 2015-06-04
+---------------
+
+Upgrade to prompt_toolkit 0.39
+
+0.11: 2015-05-31
+---------------
+
+New features:
+- Upgrade to prompt-toolkit 0.38.
+- Upgrade to Jedi 0.9.0
+- Fixed default globals for repl (correct __name, __builtins__, etc...)
+- Display deprecation warnings in the REPL.
+- Added configuration support.
+- Added asyncio-ssh-python-embed example.
+
+
+0.10: 2015-05-11
+---------------
+
+Upgrade to prompt-toolkit 0.37.
+
+
+0.9: 2015-05-07
+---------------
+
+Upgrade to prompt-toolkit 0.35.
+
+
+0.8: 2015-04-26
+---------------
+
+Fixed:
+- eval() doesn't run using unicode_literals anymore.
+- Upgrade to prompt-toolkit 0.34.
+
+
+0.7: 2015-04-25
+---------------
+
+Fixed:
+- Upgrade to prompt-toolkit 0.33.
+
+New features:
+- Added complete_while_typing option.
+
+
+0.6: 2015-04-22
+---------------
+
+Fixed:
+- Upgrade to prompt-toolkit 0.32 which has many new features.
+
+Changes:
+- Removal of tab pages + clean up.
+- Pressing enter twice will now always automatically execute the input.
+- Improved Control-D key binding.
+- Hide docstring by default.
+
+
+0.5: 2015-01-30
+---------------
+
+Fixed:
+- Tab autocompletion on first line.
+- Upgrade to prompt-toolkit 0.31
+
+New features:
+- Simple autocompletion for IPython magics.
+
+0.4: 2015-01-26
+---------------
+
+Fixed:
+- Upgrade to prompt-toolkit 0.30
+
+0.3: 2015-01-25
+---------------
+
+Fixed:
+- Upgrade to prompt-toolkit 0.28
+
+0.2: 2015-01-25
+---------------
+
+Moved ptpython code from prompt-toolkit inside this repository.
+
+0.1: 2014-09-29
+---------------
+
+Initial ptpython version. (Source code was still in the
+prompt-toolkit repository itself.)

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2015, Jonathan Slenders
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the {organization} nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,3 @@
+include *rst LICENSE CHANGELOG MANIFEST.in
+recursive-include examples *.py
+prune examples/sample?/build

README.rst

@@ -0,0 +1,245 @@
+ptpython
+========
+
+|Build Status|  |PyPI|  |License|
+
+*A better Python REPL*
+
+::
+
+    pip install ptpython
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/example1.png
+
+Ptpython is an advanced Python REPL. It should work on all
+Python versions from 2.6 up to 3.9 and work cross platform (Linux,
+BSD, OS X and Windows).
+
+Note: this version of ptpython requires at least Python 3.6. Install ptpython
+2.0.5 for older Python versions.
+
+
+Installation
+************
+
+Install it using pip:
+
+::
+
+    pip install ptpython
+
+Start it by typing ``ptpython``.
+
+
+Features
+********
+
+- Syntax highlighting.
+- Multiline editing (the up arrow works).
+- Autocompletion.
+- Mouse support. [1]
+- Support for color schemes.
+- Support for `bracketed paste <https://cirw.in/blog/bracketed-paste>`_ [2].
+- Both Vi and Emacs key bindings.
+- Support for double width (Chinese) characters.
+- ... and many other things.
+
+
+[1] Disabled by default. (Enable in the menu.)
+
+[2] If the terminal supports it (most terminals do), this allows pasting
+without going into paste mode. It will keep the indentation.
+
+__pt_repr__: A nicer repr with colors
+*************************************
+
+When classes implement a ``__pt_repr__`` method, this will be used instead of
+``__repr__`` for printing. Any `prompt_toolkit "formatted text"
+<https://python-prompt-toolkit.readthedocs.io/en/master/pages/printing_text.html>`_
+can be returned from here. In order to avoid writing a ``__repr__`` as well,
+the ``ptpython.utils.ptrepr_to_repr`` decorator can be applied. For instance:
+
+.. code:: python
+
+    from ptpython.utils import ptrepr_to_repr
+    from prompt_toolkit.formatted_text import HTML
+
+    @ptrepr_to_repr
+    class MyClass:
+        def __pt_repr__(self):
+            return HTML('<yellow>Hello world!</yellow>')
+
+More screenshots
+****************
+
+The configuration menu:
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/ptpython-menu.png
+
+The history page and its help:
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/ptpython-history-help.png
+
+Autocompletion:
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/file-completion.png
+
+
+Embedding the REPL
+******************
+
+Embedding the REPL in any Python application is easy:
+
+.. code:: python
+
+    from ptpython.repl import embed
+    embed(globals(), locals())
+
+You can make ptpython your default Python REPL by creating a `PYTHONSTARTUP file
+<https://docs.python.org/3/tutorial/appendix.html#the-interactive-startup-file>`_ containing code
+like this:
+
+.. code:: python
+
+   import sys
+   try:
+       from ptpython.repl import embed
+   except ImportError:
+       print("ptpython is not available: falling back to standard prompt")
+   else:
+       sys.exit(embed(globals(), locals()))
+
+
+Multiline editing
+*****************
+
+Multi-line editing mode will automatically turn on when you press enter after a
+colon.
+
+To execute the input in multi-line mode, you can either press ``Alt+Enter``, or
+``Esc`` followed by ``Enter``. (If you want the first to work in the OS X
+terminal, you have to check the "Use option as meta key" checkbox in your
+terminal settings. For iTerm2, you have to check "Left option acts as +Esc" in
+the options.)
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/multiline.png
+
+
+Syntax validation
+*****************
+
+Before execution, ``ptpython`` will see whether the input is syntactically
+correct Python code. If not, it will show a warning, and move the cursor to the
+error.
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/validation.png
+
+
+Additional features
+*******************
+
+Running system commands: Press ``Meta-!`` in Emacs mode or just ``!`` in Vi
+navigation mode to see the "Shell command" prompt. There you can enter system
+commands without leaving the REPL.
+
+Selecting text: Press ``Control+Space`` in Emacs mode or ``V`` (major V) in Vi
+navigation mode.
+
+
+Configuration
+*************
+
+It is possible to create a ``config.py`` file to customize configuration.
+ptpython will look in an appropriate platform-specific directory via `appdirs
+<https://pypi.org/project/appdirs/>`. See the ``appdirs`` documentation for the
+precise location for your platform. A ``PTPYTHON_CONFIG_HOME`` environment
+variable, if set, can also be used to explicitly override where configuration
+is looked for.
+
+Have a look at this example to see what is possible:
+`config.py <https://github.com/jonathanslenders/ptpython/blob/master/examples/ptpython_config/config.py>`_
+
+
+IPython support
+***************
+
+Run ``ptipython`` (prompt_toolkit - IPython), to get a nice interactive shell
+with all the power that IPython has to offer, like magic functions and shell
+integration. Make sure that IPython has been installed. (``pip install
+ipython``)
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/ipython.png
+
+This is also available for embedding:
+
+.. code:: python
+
+    from ptpython.ipython.repl import embed
+    embed(globals(), locals())
+
+
+Django support
+**************
+
+`django-extensions <https://github.com/django-extensions/django-extensions>`_
+has a ``shell_plus`` management command. When ``ptpython`` has been installed,
+it will by default use ``ptpython`` or ``ptipython``.
+
+
+PDB
+***
+
+There is an experimental PDB replacement: `ptpdb
+<https://github.com/jonathanslenders/ptpdb>`_.
+
+
+Windows support
+***************
+
+``prompt_toolkit`` and ``ptpython`` works better on Linux and OS X than on
+Windows. Some things might not work, but it is usable:
+
+.. image :: https://github.com/jonathanslenders/ptpython/raw/master/docs/images/windows.png
+
+
+FAQ
+***
+
+**Q**: The ``Ctrl-S`` forward search doesn't work and freezes my terminal.
+
+**A**: Try to run ``stty -ixon`` in your terminal to disable flow control.
+
+**Q**: The ``Meta``-key doesn't work.
+
+**A**: For some terminals you have to enable the Alt-key to act as meta key, but you 
+can also type ``Escape`` before any key instead.
+
+
+Alternatives
+************
+
+- `BPython <http://bpython-interpreter.org/downloads.html>`_
+- `IPython <https://ipython.org/>`_
+
+If you find another alternative, you can create an issue and we'll list it
+here. If you find a nice feature somewhere that is missing in ``ptpython``,
+also create a GitHub issue and maybe we'll implement it.
+
+
+Special thanks to
+*****************
+
+- `Pygments <http://pygments.org/>`_: Syntax highlighter.
+- `Jedi <http://jedi.jedidjah.ch/en/latest/>`_: Autocompletion library.
+- `wcwidth <https://github.com/jquast/wcwidth>`_: Determine columns needed for a wide characters.
+- `prompt_toolkit <http://github.com/jonathanslenders/python-prompt-toolkit>`_ for the interface.
+
+.. |Build Status| image:: https://api.travis-ci.org/prompt-toolkit/ptpython.svg?branch=master
+    :target: https://travis-ci.org/prompt-toolkit/ptpython#
+
+.. |License| image:: https://img.shields.io/github/license/prompt-toolkit/ptpython.svg
+    :target: https://github.com/prompt-toolkit/ptpython/blob/master/LICENSE
+
+.. |PyPI| image:: https://pypip.in/version/ptpython/badge.svg
+    :target: https://pypi.python.org/pypi/ptpython/
+    :alt: Latest Version

docs/concurrency-challenges.rst

@@ -0,0 +1,91 @@
+
+Concurrency-related challenges regarding embedding of ptpython in asyncio code
+==============================================================================
+
+Things we want to be possible
+-----------------------------
+
+- Embed blocking ptpython in non-asyncio code (the normal use case).
+- Embed blocking ptpython in asyncio code (the event loop will block).
+- Embed awaitable ptpython in asyncio code (the loop will continue).
+- React to resize events (SIGWINCH).
+- Support top-level await.
+- Be able to patch_stdout, so that logging messages from another thread will be
+  printed above the prompt.
+- It should be possible to handle `KeyboardInterrupt` during evaluation of an
+  expression.
+- The "eval" should happen in the same thread from where embed() was called.
+
+
+Limitations of asyncio/python
+-----------------------------
+
+- We can only listen to SIGWINCH signal (resize) events in the main thread.
+
+- Usage of Control-C for triggering a `KeyboardInterrupt` only works for code
+  running in the main thread. (And only if the terminal was not set in raw
+  input mode).
+
+- Spawning a new event loop from within a coroutine, that's being executed in
+  an existing event loop is not allowed in asyncio. We can however spawn any
+  event loop in a separate thread, and wait for that thread to finish.
+
+- For patch_stdout to work correctly, we have to know what prompt_toolkit
+  application is running on the terminal, then tell that application to print
+  the output and redraw itself.
+
+
+Additional challenges for IPython
+---------------------------------
+
+IPython supports integration of 3rd party event loops (for various GUI
+toolkits). These event loops are supposed to continue running while we are
+prompting for input. In an asyncio environment, it means that there are
+situations where we have to juggle three event loops:
+
+- The asyncio loop in which the code was embedded.
+- The asyncio loop from the prompt.
+- The 3rd party GUI loop. 
+
+Approach taken in ptpython 3.0.11
+---------------------------------
+
+For ptpython, the most reliable solution is to to run the prompt_toolkit input
+prompt in a separate background thread. This way it can use its own asyncio
+event loop without ever having to interfere with whatever runs in the main
+thread.
+
+Then, depending on how we embed, we do the following:
+When a normal blocking embed is used:
+    * We start the UI thread for the input, and do a blocking wait on
+      `thread.join()` here.
+    * The "eval" happens in the main thread.
+    * The "print" happens also in the main thread. Unless a pager is shown,
+      which is also a prompt_toolkit application, then the pager itself is runs
+      also in another thread, similar to the way we do the input.
+
+When an awaitable embed is used, for embedding in a coroutine, but having the
+event loop continue:
+    * We run the input method from the blocking embed in an asyncio executor
+      and do an `await loop.run_in_excecutor(...)`.
+    * The "eval" happens again in the main thread.
+    * "print" is also similar, except that the pager code (if used) runs in an
+      executor too.
+
+This means that the prompt_toolkit application code will always run in a
+different thread. It means it won't be able to respond to SIGWINCH (window
+resize events), but prompt_toolkit's 3.0.11 has now terminal size polling which
+solves this.
+
+Control-C key presses won't interrupt the main thread while we wait for input,
+because the prompt_toolkit application turns the terminal in raw mode, while
+it's reading, which means that it will receive control-c key presses as raw
+data in its own thread.
+
+Top-level await works in most situations as expected.
+- If a blocking embed is used. We execute ``loop.run_until_complete(code)``.
+  This assumes that the blocking embed is not used in a coroutine of a running
+  event loop, otherwise, this will attempt to start a nested event loop, which
+  asyncio does not support. In that case we will get an exception.
+- If an awaitable embed is used. We literally execute ``await code``. This will
+  integrate nicely in the current event loop.

examples/asyncio-python-embed.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""
+(Python >3.3)
+
+This is an example of how we can embed a Python REPL into an asyncio
+application. In this example, we have one coroutine that runs in the
+background, prints some output and alters a global state. The REPL, which runs
+inside another coroutine can access and change this global state, interacting
+with the running asyncio application.
+The ``patch_stdout`` option makes sure that when another coroutine is writing
+to stdout, it won't break the input line, but instead writes nicely above the
+prompt.
+"""
+import asyncio
+
+from ptpython.repl import embed
+
+loop = asyncio.get_event_loop()
+counter = [0]
+
+
+async def print_counter():
+    """
+    Coroutine that prints counters and saves it in a global variable.
+    """
+    while True:
+        print("Counter: %i" % counter[0])
+        counter[0] += 1
+        await asyncio.sleep(3)
+
+
+async def interactive_shell():
+    """
+    Coroutine that starts a Python REPL from which we can access the global
+    counter variable.
+    """
+    print(
+        'You should be able to read and update the "counter[0]" variable from this shell.'
+    )
+    try:
+        await embed(globals=globals(), return_asyncio_coroutine=True, patch_stdout=True)
+    except EOFError:
+        # Stop the loop when quitting the repl. (Ctrl-D press.)
+        loop.stop()
+
+
+def main():
+    asyncio.ensure_future(print_counter())
+    asyncio.ensure_future(interactive_shell())
+
+    loop.run_forever()
+    loop.close()
+
+
+if __name__ == "__main__":
+    main()

examples/asyncio-ssh-python-embed.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Example of running the Python REPL through an SSH connection in an asyncio process.
+This requires Python 3, asyncio and asyncssh.
+
+Run this example and then SSH to localhost, port 8222.
+"""
+import asyncio
+import logging
+
+import asyncssh
+
+from ptpython.contrib.asyncssh_repl import ReplSSHServerSession
+
+logging.basicConfig()
+logging.getLogger().setLevel(logging.INFO)
+
+
+class MySSHServer(asyncssh.SSHServer):
+    """
+    Server without authentication, running `ReplSSHServerSession`.
+    """
+
+    def __init__(self, get_namespace):
+        self.get_namespace = get_namespace
+
+    def begin_auth(self, username):
+        # No authentication.
+        return False
+
+    def session_requested(self):
+        return ReplSSHServerSession(self.get_namespace)
+
+
+def main(port=8222):
+    """
+    Example that starts the REPL through an SSH server.
+    """
+    loop = asyncio.get_event_loop()
+
+    # Namespace exposed in the REPL.
+    environ = {"hello": "world"}
+
+    # Start SSH server.
+    def create_server():
+        return MySSHServer(lambda: environ)
+
+    print("Listening on :%i" % port)
+    print('To connect, do "ssh localhost -p %i"' % port)
+
+    loop.run_until_complete(
+        asyncssh.create_server(
+            create_server, "", port, server_host_keys=["/etc/ssh/ssh_host_dsa_key"]
+        )
+    )
+
+    # Run eventloop.
+    loop.run_forever()
+
+
+if __name__ == "__main__":
+    main()

examples/ptpython_config/config.py

@@ -0,0 +1,198 @@
+"""
+Configuration example for ``ptpython``.
+
+Copy this file to $XDG_CONFIG_HOME/ptpython/config.py
+On Linux, this is: ~/.config/ptpython/config.py
+"""
+from prompt_toolkit.filters import ViInsertMode
+from prompt_toolkit.key_binding.key_processor import KeyPress
+from prompt_toolkit.keys import Keys
+from prompt_toolkit.styles import Style
+
+from ptpython.layout import CompletionVisualisation
+
+__all__ = ["configure"]
+
+
+def configure(repl):
+    """
+    Configuration method. This is called during the start-up of ptpython.
+
+    :param repl: `PythonRepl` instance.
+    """
+    # Show function signature (bool).
+    repl.show_signature = True
+
+    # Show docstring (bool).
+    repl.show_docstring = False
+
+    # Show the "[Meta+Enter] Execute" message when pressing [Enter] only
+    # inserts a newline instead of executing the code.
+    repl.show_meta_enter_message = True
+
+    # Show completions. (NONE, POP_UP, MULTI_COLUMN or TOOLBAR)
+    repl.completion_visualisation = CompletionVisualisation.POP_UP
+
+    # When CompletionVisualisation.POP_UP has been chosen, use this
+    # scroll_offset in the completion menu.
+    repl.completion_menu_scroll_offset = 0
+
+    # Show line numbers (when the input contains multiple lines.)
+    repl.show_line_numbers = False
+
+    # Show status bar.
+    repl.show_status_bar = True
+
+    # When the sidebar is visible, also show the help text.
+    repl.show_sidebar_help = True
+
+    # Swap light/dark colors on or off
+    repl.swap_light_and_dark = False
+
+    # Highlight matching parethesis.
+    repl.highlight_matching_parenthesis = True
+
+    # Line wrapping. (Instead of horizontal scrolling.)
+    repl.wrap_lines = True
+
+    # Mouse support.
+    repl.enable_mouse_support = True
+
+    # Complete while typing. (Don't require tab before the
+    # completion menu is shown.)
+    repl.complete_while_typing = True
+
+    # Fuzzy and dictionary completion.
+    repl.enable_fuzzy_completion = False
+    repl.enable_dictionary_completion = False
+
+    # Vi mode.
+    repl.vi_mode = False
+
+    # Paste mode. (When True, don't insert whitespace after new line.)
+    repl.paste_mode = False
+
+    # Use the classic prompt. (Display '>>>' instead of 'In [1]'.)
+    repl.prompt_style = "classic"  # 'classic' or 'ipython'
+
+    # Don't insert a blank line after the output.
+    repl.insert_blank_line_after_output = False
+
+    # History Search.
+    # When True, going back in history will filter the history on the records
+    # starting with the current input. (Like readline.)
+    # Note: When enable, please disable the `complete_while_typing` option.
+    #       otherwise, when there is a completion available, the arrows will
+    #       browse through the available completions instead of the history.
+    repl.enable_history_search = False
+
+    # Enable auto suggestions. (Pressing right arrow will complete the input,
+    # based on the history.)
+    repl.enable_auto_suggest = False
+
+    # Enable open-in-editor. Pressing C-x C-e in emacs mode or 'v' in
+    # Vi navigation mode will open the input in the current editor.
+    repl.enable_open_in_editor = True
+
+    # Enable system prompt. Pressing meta-! will display the system prompt.
+    # Also enables Control-Z suspend.
+    repl.enable_system_bindings = True
+
+    # Ask for confirmation on exit.
+    repl.confirm_exit = True
+
+    # Enable input validation. (Don't try to execute when the input contains
+    # syntax errors.)
+    repl.enable_input_validation = True
+
+    # Use this colorscheme for the code.
+    repl.use_code_colorscheme("default")
+    # repl.use_code_colorscheme("pastie")
+
+    # Set color depth (keep in mind that not all terminals support true color).
+
+    # repl.color_depth = "DEPTH_1_BIT"  # Monochrome.
+    # repl.color_depth = "DEPTH_4_BIT"  # ANSI colors only.
+    repl.color_depth = "DEPTH_8_BIT"  # The default, 256 colors.
+    # repl.color_depth = "DEPTH_24_BIT"  # True color.
+
+    # Min/max brightness
+    repl.min_brightness = 0.0  # Increase for dark terminal backgrounds.
+    repl.max_brightness = 1.0  # Decrease for light terminal backgrounds.
+
+    # Syntax.
+    repl.enable_syntax_highlighting = True
+
+    # Get into Vi navigation mode at startup
+    repl.vi_start_in_navigation_mode = False
+
+    # Preserve last used Vi input mode between main loop iterations
+    repl.vi_keep_last_used_mode = False
+
+    # Install custom colorscheme named 'my-colorscheme' and use it.
+    """
+    repl.install_ui_colorscheme("my-colorscheme", Style.from_dict(_custom_ui_colorscheme))
+    repl.use_ui_colorscheme("my-colorscheme")
+    """
+
+    # Add custom key binding for PDB.
+    """
+    @repl.add_key_binding("c-b")
+    def _(event):
+        " Pressing Control-B will insert "pdb.set_trace()" "
+        event.cli.current_buffer.insert_text("\nimport pdb; pdb.set_trace()\n")
+    """
+
+    # Typing ControlE twice should also execute the current command.
+    # (Alternative for Meta-Enter.)
+    """
+    @repl.add_key_binding("c-e", "c-e")
+    def _(event):
+        event.current_buffer.validate_and_handle()
+    """
+
+    # Typing 'jj' in Vi Insert mode, should send escape. (Go back to navigation
+    # mode.)
+    """
+    @repl.add_key_binding("j", "j", filter=ViInsertMode())
+    def _(event):
+        " Map 'jj' to Escape. "
+        event.cli.key_processor.feed(KeyPress("escape"))
+    """
+
+    # Custom key binding for some simple autocorrection while typing.
+    """
+    corrections = {
+        "impotr": "import",
+        "pritn": "print",
+    }
+
+    @repl.add_key_binding(" ")
+    def _(event):
+        " When a space is pressed. Check & correct word before cursor. "
+        b = event.cli.current_buffer
+        w = b.document.get_word_before_cursor()
+
+        if w is not None:
+            if w in corrections:
+                b.delete_before_cursor(count=len(w))
+                b.insert_text(corrections[w])
+
+        b.insert_text(" ")
+    """
+
+    # Add a custom title to the status bar. This is useful when ptpython is
+    # embedded in other applications.
+    """
+    repl.title = "My custom prompt."
+    """
+
+
+# Custom colorscheme for the UI. See `ptpython/layout.py` and
+# `ptpython/style.py` for all possible tokens.
+_custom_ui_colorscheme = {
+    # Blue prompt.
+    "prompt": "bg:#eeeeff #000000 bold",
+    # Make the status toolbar red.
+    "status-toolbar": "bg:#ff0000 #000000",
+}

examples/python-embed-with-custom-prompt.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""
+Example of embedding a Python REPL, and setting a custom prompt.
+"""
+from prompt_toolkit.formatted_text import HTML
+
+from ptpython.prompt_style import PromptStyle
+from ptpython.repl import embed
+
+
+def configure(repl):
+    # Probably, the best is to add a new PromptStyle to `all_prompt_styles` and
+    # activate it. This way, the other styles are still selectable from the
+    # menu.
+    class CustomPrompt(PromptStyle):
+        def in_prompt(self):
+            return HTML("<ansigreen>Input[%s]</ansigreen>: ") % (
+                repl.current_statement_index,
+            )
+
+        def in2_prompt(self, width):
+            return "...: ".rjust(width)
+
+        def out_prompt(self):
+            return HTML("<ansired>Result[%s]</ansired>: ") % (
+                repl.current_statement_index,
+            )
+
+    repl.all_prompt_styles["custom"] = CustomPrompt()
+    repl.prompt_style = "custom"
+
+
+def main():
+    embed(globals(), locals(), configure=configure)
+
+
+if __name__ == "__main__":
+    main()

examples/python-embed.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+"""
+"""
+from ptpython.repl import embed
+
+
+def main():
+    embed(globals(), locals(), vi_mode=False)
+
+
+if __name__ == "__main__":
+    main()

examples/python-input.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+"""
+"""
+from ptpython.python_input import PythonInput
+
+
+def main():
+    prompt = PythonInput()
+
+    text = prompt.app.run()
+    print("You said: " + text)
+
+
+if __name__ == "__main__":
+    main()

examples/ssh-and-telnet-embed.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+Serve a ptpython console using both telnet and ssh.
+
+Thanks to Vincent Michel for this!
+https://gist.github.com/vxgmichel/7685685b3e5ead04ada4a3ba75a48eef
+"""
+
+import asyncio
+import pathlib
+
+import asyncssh
+from prompt_toolkit import print_formatted_text
+from prompt_toolkit.contrib.ssh.server import PromptToolkitSSHServer
+from prompt_toolkit.contrib.telnet.server import TelnetServer
+
+from ptpython.repl import embed
+
+
+def ensure_key(filename="ssh_host_key"):
+    path = pathlib.Path(filename)
+    if not path.exists():
+        rsa_key = asyncssh.generate_private_key("ssh-rsa")
+        path.write_bytes(rsa_key.export_private_key())
+    return str(path)
+
+
+async def interact(connection=None):
+    global_dict = {**globals(), "print": print_formatted_text}
+    await embed(return_asyncio_coroutine=True, globals=global_dict)
+
+
+async def main(ssh_port=8022, telnet_port=8023):
+    ssh_server = PromptToolkitSSHServer(interact=interact)
+    await asyncssh.create_server(
+        lambda: ssh_server, "", ssh_port, server_host_keys=[ensure_key()]
+    )
+    print(f"Running ssh server on port {ssh_port}...")
+
+    telnet_server = TelnetServer(interact=interact, port=telnet_port)
+    telnet_server.start()
+    print(f"Running telnet server on port {telnet_port}...")
+
+    while True:
+        await asyncio.sleep(60)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/test-cases/ptpython-in-other-thread.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Example of running ptpython in another thread.
+
+(For testing whether it's working fine if it's not embedded in the main
+thread.)
+"""
+import threading
+
+from ptpython.repl import embed
+
+
+def in_thread():
+    embed(globals(), locals(), vi_mode=False)
+
+
+def main():
+    th = threading.Thread(target=in_thread)
+    th.start()
+    th.join()
+
+
+if __name__ == "__main__":
+    main()

mypy.ini

@@ -0,0 +1,6 @@
+[mypy]
+ignore_missing_imports = True
+no_implicit_optional = True
+platform = win32
+strict_equality = True
+strict_optional = True

ptpython/__init__.py

@@ -0,0 +1,3 @@
+from .repl import embed
+
+__all__ = ["embed"]

ptpython/__main__.py

@@ -0,0 +1,6 @@
+"""
+Make `python -m ptpython` an alias for running `./ptpython`.
+"""
+from .entry_points.run_ptpython import run
+
+run()

ptpython/completer.py

@@ -0,0 +1,650 @@
+import ast
+import collections.abc as collections_abc
+import inspect
+import keyword
+import re
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional
+
+from prompt_toolkit.completion import (
+    CompleteEvent,
+    Completer,
+    Completion,
+    PathCompleter,
+)
+from prompt_toolkit.contrib.completers.system import SystemCompleter
+from prompt_toolkit.contrib.regular_languages.compiler import compile as compile_grammar
+from prompt_toolkit.contrib.regular_languages.completion import GrammarCompleter
+from prompt_toolkit.document import Document
+from prompt_toolkit.formatted_text import fragment_list_to_text, to_formatted_text
+
+from ptpython.utils import get_jedi_script_from_document
+
+if TYPE_CHECKING:
+    from prompt_toolkit.contrib.regular_languages.compiler import _CompiledGrammar
+
+__all__ = ["PythonCompleter", "CompletePrivateAttributes", "HidePrivateCompleter"]
+
+
+class CompletePrivateAttributes(Enum):
+    """
+    Should we display private attributes in the completion pop-up?
+    """
+
+    NEVER = "NEVER"
+    IF_NO_PUBLIC = "IF_NO_PUBLIC"
+    ALWAYS = "ALWAYS"
+
+
+class PythonCompleter(Completer):
+    """
+    Completer for Python code.
+    """
+
+    def __init__(
+        self,
+        get_globals: Callable[[], dict],
+        get_locals: Callable[[], dict],
+        enable_dictionary_completion: Callable[[], bool],
+    ) -> None:
+        super().__init__()
+
+        self.get_globals = get_globals
+        self.get_locals = get_locals
+        self.enable_dictionary_completion = enable_dictionary_completion
+
+        self._system_completer = SystemCompleter()
+        self._jedi_completer = JediCompleter(get_globals, get_locals)
+        self._dictionary_completer = DictionaryCompleter(get_globals, get_locals)
+
+        self._path_completer_cache: Optional[GrammarCompleter] = None
+        self._path_completer_grammar_cache: Optional["_CompiledGrammar"] = None
+
+    @property
+    def _path_completer(self) -> GrammarCompleter:
+        if self._path_completer_cache is None:
+            self._path_completer_cache = GrammarCompleter(
+                self._path_completer_grammar,
+                {
+                    "var1": PathCompleter(expanduser=True),
+                    "var2": PathCompleter(expanduser=True),
+                },
+            )
+        return self._path_completer_cache
+
+    @property
+    def _path_completer_grammar(self) -> "_CompiledGrammar":
+        """
+        Return the grammar for matching paths inside strings inside Python
+        code.
+        """
+        # We make this lazy, because it delays startup time a little bit.
+        # This way, the grammar is build during the first completion.
+        if self._path_completer_grammar_cache is None:
+            self._path_completer_grammar_cache = self._create_path_completer_grammar()
+        return self._path_completer_grammar_cache
+
+    def _create_path_completer_grammar(self) -> "_CompiledGrammar":
+        def unwrapper(text: str) -> str:
+            return re.sub(r"\\(.)", r"\1", text)
+
+        def single_quoted_wrapper(text: str) -> str:
+            return text.replace("\\", "\\\\").replace("'", "\\'")
+
+        def double_quoted_wrapper(text: str) -> str:
+            return text.replace("\\", "\\\\").replace('"', '\\"')
+
+        grammar = r"""
+                # Text before the current string.
+                (
+                    [^'"#]                                  |  # Not quoted characters.
+                    '''  ([^'\\]|'(?!')|''(?!')|\\.])*  ''' |  # Inside single quoted triple strings
+                    "" " ([^"\\]|"(?!")|""(?!^)|\\.])* "" " |  # Inside double quoted triple strings
+
+                    \#[^\n]*(\n|$)           |  # Comment.
+                    "(?!"") ([^"\\]|\\.)*"   |  # Inside double quoted strings.
+                    '(?!'') ([^'\\]|\\.)*'      # Inside single quoted strings.
+
+                        # Warning: The negative lookahead in the above two
+                        #          statements is important. If we drop that,
+                        #          then the regex will try to interpret every
+                        #          triple quoted string also as a single quoted
+                        #          string, making this exponentially expensive to
+                        #          execute!
+                )*
+                # The current string that we're completing.
+                (
+                    ' (?P<var1>([^\n'\\]|\\.)*) |  # Inside a single quoted string.
+                    " (?P<var2>([^\n"\\]|\\.)*)    # Inside a double quoted string.
+                )
+        """
+
+        return compile_grammar(
+            grammar,
+            escape_funcs={"var1": single_quoted_wrapper, "var2": double_quoted_wrapper},
+            unescape_funcs={"var1": unwrapper, "var2": unwrapper},
+        )
+
+    def _complete_path_while_typing(self, document: Document) -> bool:
+        char_before_cursor = document.char_before_cursor
+        return bool(
+            document.text
+            and (char_before_cursor.isalnum() or char_before_cursor in "/.~")
+        )
+
+    def _complete_python_while_typing(self, document: Document) -> bool:
+        """
+        When `complete_while_typing` is set, only return completions when this
+        returns `True`.
+        """
+        text = document.text_before_cursor  # .rstrip()
+        char_before_cursor = text[-1:]
+        return bool(
+            text and (char_before_cursor.isalnum() or char_before_cursor in "_.([,")
+        )
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        """
+        Get Python completions.
+        """
+        # If the input starts with an exclamation mark. Use the system completer.
+        if document.text.lstrip().startswith("!"):
+            yield from self._system_completer.get_completions(
+                Document(
+                    text=document.text[1:], cursor_position=document.cursor_position - 1
+                ),
+                complete_event,
+            )
+            return
+
+        # Do dictionary key completions.
+        if complete_event.completion_requested or self._complete_python_while_typing(
+            document
+        ):
+            if self.enable_dictionary_completion():
+                has_dict_completions = False
+                for c in self._dictionary_completer.get_completions(
+                    document, complete_event
+                ):
+                    if c.text not in "[.":
+                        # If we get the [ or . completion, still include the other
+                        # completions.
+                        has_dict_completions = True
+                    yield c
+                if has_dict_completions:
+                    return
+
+        # Do Path completions (if there were no dictionary completions).
+        if complete_event.completion_requested or self._complete_path_while_typing(
+            document
+        ):
+            yield from self._path_completer.get_completions(document, complete_event)
+
+        # Do Jedi completions.
+        if complete_event.completion_requested or self._complete_python_while_typing(
+            document
+        ):
+            # If we are inside a string, Don't do Jedi completion.
+            if not self._path_completer_grammar.match(document.text_before_cursor):
+
+                # Do Jedi Python completions.
+                yield from self._jedi_completer.get_completions(
+                    document, complete_event
+                )
+
+
+class JediCompleter(Completer):
+    """
+    Autocompleter that uses the Jedi library.
+    """
+
+    def __init__(self, get_globals, get_locals) -> None:
+        super().__init__()
+
+        self.get_globals = get_globals
+        self.get_locals = get_locals
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        script = get_jedi_script_from_document(
+            document, self.get_locals(), self.get_globals()
+        )
+
+        if script:
+            try:
+                jedi_completions = script.complete(
+                    column=document.cursor_position_col,
+                    line=document.cursor_position_row + 1,
+                )
+            except TypeError:
+                # Issue #9: bad syntax causes completions() to fail in jedi.
+                # https://github.com/jonathanslenders/python-prompt-toolkit/issues/9
+                pass
+            except UnicodeDecodeError:
+                # Issue #43: UnicodeDecodeError on OpenBSD
+                # https://github.com/jonathanslenders/python-prompt-toolkit/issues/43
+                pass
+            except AttributeError:
+                # Jedi issue #513: https://github.com/davidhalter/jedi/issues/513
+                pass
+            except ValueError:
+                # Jedi issue: "ValueError: invalid \x escape"
+                pass
+            except KeyError:
+                # Jedi issue: "KeyError: u'a_lambda'."
+                # https://github.com/jonathanslenders/ptpython/issues/89
+                pass
+            except IOError:
+                # Jedi issue: "IOError: No such file or directory."
+                # https://github.com/jonathanslenders/ptpython/issues/71
+                pass
+            except AssertionError:
+                # In jedi.parser.__init__.py: 227, in remove_last_newline,
+                # the assertion "newline.value.endswith('\n')" can fail.
+                pass
+            except SystemError:
+                # In jedi.api.helpers.py: 144, in get_stack_at_position
+                # raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
+                pass
+            except NotImplementedError:
+                # See: https://github.com/jonathanslenders/ptpython/issues/223
+                pass
+            except Exception:
+                # Supress all other Jedi exceptions.
+                pass
+            else:
+                # Move function parameters to the top.
+                jedi_completions = sorted(
+                    jedi_completions,
+                    key=lambda jc: (
+                        # Params first.
+                        jc.type != "param",
+                        # Private at the end.
+                        jc.name.startswith("_"),
+                        # Then sort by name.
+                        jc.name_with_symbols.lower(),
+                    ),
+                )
+
+                for jc in jedi_completions:
+                    if jc.type == "function":
+                        suffix = "()"
+                    else:
+                        suffix = ""
+
+                    if jc.type == "param":
+                        suffix = "..."
+
+                    yield Completion(
+                        jc.name_with_symbols,
+                        len(jc.complete) - len(jc.name_with_symbols),
+                        display=jc.name_with_symbols + suffix,
+                        display_meta=jc.type,
+                        style=_get_style_for_jedi_completion(jc),
+                    )
+
+
+class DictionaryCompleter(Completer):
+    """
+    Experimental completer for Python dictionary keys.
+
+    Warning: This does an `eval` and `repr` on some Python expressions before
+             the cursor, which is potentially dangerous. It doesn't match on
+             function calls, so it only triggers attribute access.
+    """
+
+    def __init__(self, get_globals, get_locals):
+        super().__init__()
+
+        self.get_globals = get_globals
+        self.get_locals = get_locals
+
+        # Pattern for expressions that are "safe" to eval for auto-completion.
+        # These are expressions that contain only attribute and index lookups.
+        varname = r"[a-zA-Z_][a-zA-Z0-9_]*"
+
+        expression = rf"""
+            # Any expression safe enough to eval while typing.
+            # No operators, except dot, and only other dict lookups.
+            # Technically, this can be unsafe of course, if bad code runs
+            # in `__getattr__` or ``__getitem__``.
+            (
+                # Variable name
+                {varname}
+
+                \s*
+
+                (?:
+                    # Attribute access.
+                    \s* \. \s* {varname} \s*
+
+                    |
+
+                    # Item lookup.
+                    # (We match the square brackets. The key can be anything.
+                    # We don't care about matching quotes here in the regex.
+                    # Nested square brackets are not supported.)
+                    \s* \[ [^\[\]]+ \] \s*
+                )*
+            )
+        """
+
+        # Pattern for recognizing for-loops, so that we can provide
+        # autocompletion on the iterator of the for-loop. (According to the
+        # first item of the collection we're iterating over.)
+        self.for_loop_pattern = re.compile(
+            rf"""
+                for \s+ ([a-zA-Z0-9_]+) \s+ in \s+ {expression} \s* :
+            """,
+            re.VERBOSE,
+        )
+
+        # Pattern for matching a simple expression (for completing [ or .
+        # operators).
+        self.expression_pattern = re.compile(
+            rf"""
+                {expression}
+                $
+            """,
+            re.VERBOSE,
+        )
+
+        # Pattern for matching item lookups.
+        self.item_lookup_pattern = re.compile(
+            rf"""
+                {expression}
+
+                # Dict loopup to complete (square bracket open + start of
+                # string).
+                \[
+                \s* ([^\[\]]*)$
+            """,
+            re.VERBOSE,
+        )
+
+        # Pattern for matching attribute lookups.
+        self.attribute_lookup_pattern = re.compile(
+            rf"""
+                {expression}
+
+                # Attribute loopup to complete (dot + varname).
+                \.
+                \s* ([a-zA-Z0-9_]*)$
+            """,
+            re.VERBOSE,
+        )
+
+    def _lookup(self, expression: str, temp_locals: Dict[str, Any]) -> object:
+        """
+        Do lookup of `object_var` in the context.
+        `temp_locals` is a dictionary, used for the locals.
+        """
+        try:
+            return eval(expression.strip(), self.get_globals(), temp_locals)
+        except BaseException:
+            return None  # Many exception, like NameError can be thrown here.
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+
+        # First, find all for-loops, and assign the first item of the
+        # collections they're iterating to the iterator variable, so that we
+        # can provide code completion on the iterators.
+        temp_locals = self.get_locals().copy()
+
+        for match in self.for_loop_pattern.finditer(document.text_before_cursor):
+            varname, expression = match.groups()
+            expression_val = self._lookup(expression, temp_locals)
+
+            # We do this only for lists and tuples. Calling `next()` on any
+            # collection would create undesired side effects.
+            if isinstance(expression_val, (list, tuple)) and expression_val:
+                temp_locals[varname] = expression_val[0]
+
+        # Get all completions.
+        yield from self._get_expression_completions(
+            document, complete_event, temp_locals
+        )
+        yield from self._get_item_lookup_completions(
+            document, complete_event, temp_locals
+        )
+        yield from self._get_attribute_completions(
+            document, complete_event, temp_locals
+        )
+
+    def _do_repr(self, obj: object) -> str:
+        try:
+            return str(repr(obj))
+        except BaseException:
+            raise ReprFailedError
+
+    def eval_expression(self, document: Document, locals: Dict[str, Any]) -> object:
+        """
+        Evaluate
+        """
+        match = self.expression_pattern.search(document.text_before_cursor)
+        if match is not None:
+            object_var = match.groups()[0]
+            return self._lookup(object_var, locals)
+
+        return None
+
+    def _get_expression_completions(
+        self,
+        document: Document,
+        complete_event: CompleteEvent,
+        temp_locals: Dict[str, Any],
+    ) -> Iterable[Completion]:
+        """
+        Complete the [ or . operator after an object.
+        """
+        result = self.eval_expression(document, temp_locals)
+
+        if result is not None:
+
+            if isinstance(
+                result,
+                (list, tuple, dict, collections_abc.Mapping, collections_abc.Sequence),
+            ):
+                yield Completion("[", 0)
+
+            else:
+                # Note: Don't call `if result` here. That can fail for types
+                #       that have custom truthness checks.
+                yield Completion(".", 0)
+
+    def _get_item_lookup_completions(
+        self,
+        document: Document,
+        complete_event: CompleteEvent,
+        temp_locals: Dict[str, Any],
+    ) -> Iterable[Completion]:
+        """
+        Complete dictionary keys.
+        """
+
+        def abbr_meta(text: str) -> str:
+            " Abbreviate meta text, make sure it fits on one line. "
+            # Take first line, if multiple lines.
+            if len(text) > 20:
+                text = text[:20] + "..."
+            if "\n" in text:
+                text = text.split("\n", 1)[0] + "..."
+            return text
+
+        match = self.item_lookup_pattern.search(document.text_before_cursor)
+        if match is not None:
+            object_var, key = match.groups()
+
+            # Do lookup of `object_var` in the context.
+            result = self._lookup(object_var, temp_locals)
+
+            # If this object is a dictionary, complete the keys.
+            if isinstance(result, (dict, collections_abc.Mapping)):
+                # Try to evaluate the key.
+                key_obj = key
+                for k in [key, key + '"', key + "'"]:
+                    try:
+                        key_obj = ast.literal_eval(k)
+                    except (SyntaxError, ValueError):
+                        continue
+                    else:
+                        break
+
+                for k in result:
+                    if str(k).startswith(str(key_obj)):
+                        try:
+                            k_repr = self._do_repr(k)
+                            yield Completion(
+                                k_repr + "]",
+                                -len(key),
+                                display=f"[{k_repr}]",
+                                display_meta=abbr_meta(self._do_repr(result[k])),
+                            )
+                        except ReprFailedError:
+                            pass
+
+            # Complete list/tuple index keys.
+            elif isinstance(result, (list, tuple, collections_abc.Sequence)):
+                if not key or key.isdigit():
+                    for k in range(min(len(result), 1000)):
+                        if str(k).startswith(key):
+                            try:
+                                k_repr = self._do_repr(k)
+                                yield Completion(
+                                    k_repr + "]",
+                                    -len(key),
+                                    display=f"[{k_repr}]",
+                                    display_meta=abbr_meta(self._do_repr(result[k])),
+                                )
+                            except ReprFailedError:
+                                pass
+
+    def _get_attribute_completions(
+        self,
+        document: Document,
+        complete_event: CompleteEvent,
+        temp_locals: Dict[str, Any],
+    ) -> Iterable[Completion]:
+        """
+        Complete attribute names.
+        """
+        match = self.attribute_lookup_pattern.search(document.text_before_cursor)
+        if match is not None:
+            object_var, attr_name = match.groups()
+
+            # Do lookup of `object_var` in the context.
+            result = self._lookup(object_var, temp_locals)
+
+            names = self._sort_attribute_names(dir(result))
+
+            def get_suffix(name: str) -> str:
+                try:
+                    obj = getattr(result, name, None)
+                    if inspect.isfunction(obj):
+                        return "()"
+
+                    if isinstance(obj, dict):
+                        return "{}"
+                    if isinstance(obj, (list, tuple)):
+                        return "[]"
+                except:
+                    pass
+                return ""
+
+            for name in names:
+                if name.startswith(attr_name):
+                    suffix = get_suffix(name)
+                    yield Completion(name, -len(attr_name), display=name + suffix)
+
+    def _sort_attribute_names(self, names: List[str]) -> List[str]:
+        """
+        Sort attribute names alphabetically, but move the double underscore and
+        underscore names to the end.
+        """
+
+        def sort_key(name: str):
+            if name.startswith("__"):
+                return (2, name)  # Double underscore comes latest.
+            if name.startswith("_"):
+                return (1, name)  # Single underscore before that.
+            return (0, name)  # Other names first.
+
+        return sorted(names, key=sort_key)
+
+
+class HidePrivateCompleter(Completer):
+    """
+    Wrapper around completer that hides private fields, deponding on whether or
+    not public fields are shown.
+
+    (The reason this is implemented as a `Completer` wrapper is because this
+    way it works also with `FuzzyCompleter`.)
+    """
+
+    def __init__(
+        self,
+        completer: Completer,
+        complete_private_attributes: Callable[[], CompletePrivateAttributes],
+    ) -> None:
+        self.completer = completer
+        self.complete_private_attributes = complete_private_attributes
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+
+        completions = list(self.completer.get_completions(document, complete_event))
+        complete_private_attributes = self.complete_private_attributes()
+        hide_private = False
+
+        def is_private(completion: Completion) -> bool:
+            text = fragment_list_to_text(to_formatted_text(completion.display))
+            return text.startswith("_")
+
+        if complete_private_attributes == CompletePrivateAttributes.NEVER:
+            hide_private = True
+
+        elif complete_private_attributes == CompletePrivateAttributes.IF_NO_PUBLIC:
+            hide_private = any(not is_private(completion) for completion in completions)
+
+        if hide_private:
+            completions = [
+                completion for completion in completions if not is_private(completion)
+            ]
+
+        return completions
+
+
+class ReprFailedError(Exception):
+    " Raised when the repr() call in `DictionaryCompleter` fails. "
+
+
+try:
+    import builtins
+
+    _builtin_names = dir(builtins)
+except ImportError:  # Python 2.
+    _builtin_names = []
+
+
+def _get_style_for_jedi_completion(jedi_completion) -> str:
+    """
+    Return completion style to use for this name.
+    """
+    name = jedi_completion.name_with_symbols
+
+    if jedi_completion.type == "param":
+        return "class:completion.param"
+
+    if name in _builtin_names:
+        return "class:completion.builtin"
+
+    if keyword.iskeyword(name):
+        return "class:completion.keyword"
+
+    return ""

ptpython/contrib/asyncssh_repl.py

@@ -0,0 +1,119 @@
+"""
+Tool for embedding a REPL inside a Python 3 asyncio process.
+See ./examples/asyncio-ssh-python-embed.py for a demo.
+
+Note that the code in this file is Python 3 only. However, we
+should make sure not to use Python 3-only syntax, because this
+package should be installable in Python 2 as well!
+"""
+import asyncio
+from typing import Any, Optional, TextIO, cast
+
+import asyncssh
+from prompt_toolkit.data_structures import Size
+from prompt_toolkit.input import create_pipe_input
+from prompt_toolkit.output.vt100 import Vt100_Output
+
+from ptpython.python_input import _GetNamespace
+from ptpython.repl import PythonRepl
+
+__all__ = ["ReplSSHServerSession"]
+
+
+class ReplSSHServerSession(asyncssh.SSHServerSession):
+    """
+    SSH server session that runs a Python REPL.
+
+    :param get_globals: callable that returns the current globals.
+    :param get_locals: (optional) callable that returns the current locals.
+    """
+
+    def __init__(
+        self, get_globals: _GetNamespace, get_locals: Optional[_GetNamespace] = None
+    ) -> None:
+        self._chan: Any = None
+
+        def _globals() -> dict:
+            data = get_globals()
+            data.setdefault("print", self._print)
+            return data
+
+        # PipInput object, for sending input in the CLI.
+        # (This is something that we can use in the prompt_toolkit event loop,
+        # but still write date in manually.)
+        self._input_pipe = create_pipe_input()
+
+        # Output object. Don't render to the real stdout, but write everything
+        # in the SSH channel.
+        class Stdout:
+            def write(s, data: str) -> None:
+                if self._chan is not None:
+                    data = data.replace("\n", "\r\n")
+                    self._chan.write(data)
+
+            def flush(s) -> None:
+                pass
+
+        self.repl = PythonRepl(
+            get_globals=_globals,
+            get_locals=get_locals or _globals,
+            input=self._input_pipe,
+            output=Vt100_Output(cast(TextIO, Stdout()), self._get_size),
+        )
+
+        # Disable open-in-editor and system prompt. Because it would run and
+        # display these commands on the server side, rather than in the SSH
+        # client.
+        self.repl.enable_open_in_editor = False
+        self.repl.enable_system_bindings = False
+
+    def _get_size(self) -> Size:
+        """
+        Callable that returns the current `Size`, required by Vt100_Output.
+        """
+        if self._chan is None:
+            return Size(rows=20, columns=79)
+        else:
+            width, height, pixwidth, pixheight = self._chan.get_terminal_size()
+            return Size(rows=height, columns=width)
+
+    def connection_made(self, chan):
+        """
+        Client connected, run repl in coroutine.
+        """
+        self._chan = chan
+
+        # Run REPL interface.
+        f = asyncio.ensure_future(self.repl.run_async())
+
+        # Close channel when done.
+        def done(_) -> None:
+            chan.close()
+            self._chan = None
+
+        f.add_done_callback(done)
+
+    def shell_requested(self) -> bool:
+        return True
+
+    def terminal_size_changed(self, width, height, pixwidth, pixheight):
+        """
+        When the terminal size changes, report back to CLI.
+        """
+        self.repl.app._on_resize()
+
+    def data_received(self, data, datatype):
+        """
+        When data is received, send to inputstream of the CLI and repaint.
+        """
+        self._input_pipe.send(data)
+
+    def _print(self, *data, sep=" ", end="\n", file=None) -> None:
+        """
+        Alternative 'print' function that prints back into the SSH channel.
+        """
+        # Pop keyword-only arguments. (We cannot use the syntax from the
+        # signature. Otherwise, Python2 will give a syntax error message when
+        # installing.)
+        data = sep.join(map(str, data))
+        self._chan.write(data + end)

ptpython/entry_points/run_ptipython.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+import os
+import sys
+
+from .run_ptpython import create_parser, get_config_and_history_file
+
+
+def run(user_ns=None):
+    a = create_parser().parse_args()
+
+    config_file, history_file = get_config_and_history_file(a)
+
+    # If IPython is not available, show message and exit here with error status
+    # code.
+    try:
+        import IPython
+    except ImportError:
+        print("IPython not found. Please install IPython (pip install ipython).")
+        sys.exit(1)
+    else:
+        from ptpython.ipython import embed
+        from ptpython.repl import enable_deprecation_warnings, run_config
+
+    # Add the current directory to `sys.path`.
+    if sys.path[0] != "":
+        sys.path.insert(0, "")
+
+    # When a file has been given, run that, otherwise start the shell.
+    if a.args and not a.interactive:
+        sys.argv = a.args
+        path = a.args[0]
+        with open(path, "rb") as f:
+            code = compile(f.read(), path, "exec")
+            exec(code, {})
+    else:
+        enable_deprecation_warnings()
+
+        # Create an empty namespace for this interactive shell. (If we don't do
+        # that, all the variables from this function will become available in
+        # the IPython shell.)
+        if user_ns is None:
+            user_ns = {}
+
+        # Startup path
+        startup_paths = []
+        if "PYTHONSTARTUP" in os.environ:
+            startup_paths.append(os.environ["PYTHONSTARTUP"])
+
+        # --interactive
+        if a.interactive:
+            startup_paths.append(a.args[0])
+            sys.argv = a.args
+
+        # exec scripts from startup paths
+        for path in startup_paths:
+            if os.path.exists(path):
+                with open(path, "rb") as f:
+                    code = compile(f.read(), path, "exec")
+                    exec(code, user_ns, user_ns)
+            else:
+                print("File not found: {}\n\n".format(path))
+                sys.exit(1)
+
+        # Apply config file
+        def configure(repl):
+            if os.path.exists(config_file):
+                run_config(repl, config_file)
+
+        # Run interactive shell.
+        embed(
+            vi_mode=a.vi,
+            history_filename=history_file,
+            configure=configure,
+            user_ns=user_ns,
+            title="IPython REPL (ptipython)",
+        )
+
+
+if __name__ == "__main__":
+    run()

ptpython/entry_points/run_ptpython.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+"""
+ptpython: Interactive Python shell.
+
+positional arguments:
+  args                  Script and arguments
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --vi                  Enable Vi key bindings
+  -i, --interactive     Start interactive shell after executing this file.
+  --light-bg            Run on a light background (use dark colors for text).
+  --dark-bg             Run on a dark background (use light colors for text).
+  --config-file CONFIG_FILE
+                        Location of configuration file.
+  --history-file HISTORY_FILE
+                        Location of history file.
+  -V, --version         show program's version number and exit
+
+environment variables:
+  PTPYTHON_CONFIG_HOME: a configuration directory to use
+  PYTHONSTARTUP: file executed on interactive startup (no default)
+"""
+import argparse
+import os
+import pathlib
+import sys
+from textwrap import dedent
+from typing import Tuple
+
+import appdirs
+from prompt_toolkit.formatted_text import HTML
+from prompt_toolkit.shortcuts import print_formatted_text
+
+from ptpython.repl import embed, enable_deprecation_warnings, run_config
+
+try:
+    from importlib import metadata
+except ImportError:
+    import importlib_metadata as metadata  # type: ignore
+
+
+__all__ = ["create_parser", "get_config_and_history_file", "run"]
+
+
+class _Parser(argparse.ArgumentParser):
+    def print_help(self):
+        super().print_help()
+        print(
+            dedent(
+                """
+                environment variables:
+                  PTPYTHON_CONFIG_HOME: a configuration directory to use
+                  PYTHONSTARTUP: file executed on interactive startup (no default)
+                """,
+            ).rstrip(),
+        )
+
+
+def create_parser() -> _Parser:
+    parser = _Parser(description="ptpython: Interactive Python shell.")
+    parser.add_argument("--vi", action="store_true", help="Enable Vi key bindings")
+    parser.add_argument(
+        "-i",
+        "--interactive",
+        action="store_true",
+        help="Start interactive shell after executing this file.",
+    )
+    parser.add_argument(
+        "--light-bg",
+        action="store_true",
+        help="Run on a light background (use dark colors for text).",
+    ),
+    parser.add_argument(
+        "--dark-bg",
+        action="store_true",
+        help="Run on a dark background (use light colors for text).",
+    ),
+    parser.add_argument(
+        "--config-file", type=str, help="Location of configuration file."
+    )
+    parser.add_argument("--history-file", type=str, help="Location of history file.")
+    parser.add_argument(
+        "-V",
+        "--version",
+        action="version",
+        version=metadata.version("ptpython"),  # type: ignore
+    )
+    parser.add_argument("args", nargs="*", help="Script and arguments")
+    return parser
+
+
+def get_config_and_history_file(namespace: argparse.Namespace) -> Tuple[str, str]:
+    """
+    Check which config/history files to use, ensure that the directories for
+    these files exist, and return the config and history path.
+    """
+    config_dir = os.environ.get(
+        "PTPYTHON_CONFIG_HOME",
+        appdirs.user_config_dir("ptpython", "prompt_toolkit"),
+    )
+    data_dir = appdirs.user_data_dir("ptpython", "prompt_toolkit")
+
+    # Create directories.
+    for d in (config_dir, data_dir):
+        pathlib.Path(d).mkdir(parents=True, exist_ok=True)
+
+    # Determine config file to be used.
+    config_file = os.path.join(config_dir, "config.py")
+    legacy_config_file = os.path.join(os.path.expanduser("~/.ptpython"), "config.py")
+
+    warnings = []
+
+    # Config file
+    if namespace.config_file:
+        # Override config_file.
+        config_file = os.path.expanduser(namespace.config_file)
+
+    elif os.path.isfile(legacy_config_file):
+        # Warn about the legacy configuration file.
+        warnings.append(
+            HTML(
+                "    <i>~/.ptpython/config.py</i> is deprecated, move your configuration to <i>%s</i>\n"
+            )
+            % config_file
+        )
+        config_file = legacy_config_file
+
+    # Determine history file to be used.
+    history_file = os.path.join(data_dir, "history")
+    legacy_history_file = os.path.join(os.path.expanduser("~/.ptpython"), "history")
+
+    if namespace.history_file:
+        # Override history_file.
+        history_file = os.path.expanduser(namespace.history_file)
+
+    elif os.path.isfile(legacy_history_file):
+        # Warn about the legacy history file.
+        warnings.append(
+            HTML(
+                "    <i>~/.ptpython/history</i> is deprecated, move your history to <i>%s</i>\n"
+            )
+            % history_file
+        )
+        history_file = legacy_history_file
+
+    # Print warnings.
+    if warnings:
+        print_formatted_text(HTML("<u>Warning:</u>"))
+        for w in warnings:
+            print_formatted_text(w)
+
+    return config_file, history_file
+
+
+def run() -> None:
+    a = create_parser().parse_args()
+
+    config_file, history_file = get_config_and_history_file(a)
+
+    # Startup path
+    startup_paths = []
+    if "PYTHONSTARTUP" in os.environ:
+        startup_paths.append(os.environ["PYTHONSTARTUP"])
+
+    # --interactive
+    if a.interactive and a.args:
+        # Note that we shouldn't run PYTHONSTARTUP when -i is given.
+        startup_paths = [a.args[0]]
+        sys.argv = a.args
+
+    # Add the current directory to `sys.path`.
+    if sys.path[0] != "":
+        sys.path.insert(0, "")
+
+    # When a file has been given, run that, otherwise start the shell.
+    if a.args and not a.interactive:
+        sys.argv = a.args
+        path = a.args[0]
+        with open(path, "rb") as f:
+            code = compile(f.read(), path, "exec")
+            # NOTE: We have to pass an empty dictionary as namespace. Omitting
+            #       this argument causes imports to not be found. See issue #326.
+            exec(code, {})
+
+    # Run interactive shell.
+    else:
+        enable_deprecation_warnings()
+
+        # Apply config file
+        def configure(repl) -> None:
+            if os.path.exists(config_file):
+                run_config(repl, config_file)
+
+            # Adjust colors if dark/light background flag has been given.
+            if a.light_bg:
+                repl.min_brightness = 0.0
+                repl.max_brightness = 0.60
+            elif a.dark_bg:
+                repl.min_brightness = 0.60
+                repl.max_brightness = 1.0
+
+        import __main__
+
+        embed(
+            vi_mode=a.vi,
+            history_filename=history_file,
+            configure=configure,
+            locals=__main__.__dict__,
+            globals=__main__.__dict__,
+            startup_paths=startup_paths,
+            title="Python REPL (ptpython)",
+        )
+
+
+if __name__ == "__main__":
+    run()

ptpython/eventloop.py

@@ -0,0 +1,71 @@
+"""
+Wrapper around the eventloop that gives some time to the Tkinter GUI to process
+events when it's loaded and while we are waiting for input at the REPL. This
+way we don't block the UI of for instance ``turtle`` and other Tk libraries.
+
+(Normally Tkinter registers it's callbacks in ``PyOS_InputHook`` to integrate
+in readline. ``prompt-toolkit`` doesn't understand that input hook, but this
+will fix it for Tk.)
+"""
+import sys
+import time
+
+__all__ = ["inputhook"]
+
+
+def _inputhook_tk(inputhook_context):
+    """
+    Inputhook for Tk.
+    Run the Tk eventloop until prompt-toolkit needs to process the next input.
+    """
+    # Get the current TK application.
+    import tkinter
+
+    import _tkinter  # Keep this imports inline!
+
+    root = tkinter._default_root
+
+    def wait_using_filehandler():
+        """
+        Run the TK eventloop until the file handler that we got from the
+        inputhook becomes readable.
+        """
+        # Add a handler that sets the stop flag when `prompt-toolkit` has input
+        # to process.
+        stop = [False]
+
+        def done(*a):
+            stop[0] = True
+
+        root.createfilehandler(inputhook_context.fileno(), _tkinter.READABLE, done)
+
+        # Run the TK event loop as long as we don't receive input.
+        while root.dooneevent(_tkinter.ALL_EVENTS):
+            if stop[0]:
+                break
+
+        root.deletefilehandler(inputhook_context.fileno())
+
+    def wait_using_polling():
+        """
+        Windows TK doesn't support 'createfilehandler'.
+        So, run the TK eventloop and poll until input is ready.
+        """
+        while not inputhook_context.input_is_ready():
+            while root.dooneevent(_tkinter.ALL_EVENTS | _tkinter.DONT_WAIT):
+                pass
+            # Sleep to make the CPU idle, but not too long, so that the UI
+            # stays responsive.
+            time.sleep(0.01)
+
+    if root is not None:
+        if hasattr(root, "createfilehandler"):
+            wait_using_filehandler()
+        else:
+            wait_using_polling()
+
+
+def inputhook(inputhook_context):
+    # Only call the real input hook when the 'Tkinter' library was loaded.
+    if "Tkinter" in sys.modules or "tkinter" in sys.modules:
+        _inputhook_tk(inputhook_context)

ptpython/filters.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING
+
+from prompt_toolkit.filters import Filter
+
+if TYPE_CHECKING:
+    from .python_input import PythonInput
+
+__all__ = ["HasSignature", "ShowSidebar", "ShowSignature", "ShowDocstring"]
+
+
+class PythonInputFilter(Filter):
+    def __init__(self, python_input: "PythonInput") -> None:
+        self.python_input = python_input
+
+    def __call__(self) -> bool:
+        raise NotImplementedError
+
+
+class HasSignature(PythonInputFilter):
+    def __call__(self) -> bool:
+        return bool(self.python_input.signatures)
+
+
+class ShowSidebar(PythonInputFilter):
+    def __call__(self) -> bool:
+        return self.python_input.show_sidebar
+
+
+class ShowSignature(PythonInputFilter):
+    def __call__(self) -> bool:
+        return self.python_input.show_signature
+
+
+class ShowDocstring(PythonInputFilter):
+    def __call__(self) -> bool:
+        return self.python_input.show_docstring

ptpython/history_browser.py

@@ -0,0 +1,648 @@
+"""
+Utility to easily select lines from the history and execute them again.
+
+`create_history_application` creates an `Application` instance that runs will
+run as a sub application of the Repl/PythonInput.
+"""
+from functools import partial
+
+from prompt_toolkit.application import Application
+from prompt_toolkit.application.current import get_app
+from prompt_toolkit.buffer import Buffer
+from prompt_toolkit.document import Document
+from prompt_toolkit.enums import DEFAULT_BUFFER
+from prompt_toolkit.filters import Condition, has_focus
+from prompt_toolkit.formatted_text.utils import fragment_list_to_text
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.layout.containers import (
+    ConditionalContainer,
+    Container,
+    Float,
+    FloatContainer,
+    HSplit,
+    ScrollOffsets,
+    VSplit,
+    Window,
+    WindowAlign,
+)
+from prompt_toolkit.layout.controls import BufferControl, FormattedTextControl
+from prompt_toolkit.layout.dimension import Dimension as D
+from prompt_toolkit.layout.layout import Layout
+from prompt_toolkit.layout.margins import Margin, ScrollbarMargin
+from prompt_toolkit.layout.processors import Processor, Transformation
+from prompt_toolkit.lexers import PygmentsLexer
+from prompt_toolkit.widgets import Frame
+from prompt_toolkit.widgets.toolbars import ArgToolbar, SearchToolbar
+from pygments.lexers import Python3Lexer as PythonLexer
+from pygments.lexers import RstLexer
+
+from ptpython.layout import get_inputmode_fragments
+
+from .utils import if_mousedown
+
+HISTORY_COUNT = 2000
+
+__all__ = ["HistoryLayout", "PythonHistory"]
+
+HELP_TEXT = """
+This interface is meant to select multiple lines from the
+history and execute them together.
+
+Typical usage
+-------------
+
+1. Move the ``cursor up`` in the history pane, until the
+   cursor is on the first desired line.
+2. Hold down the ``space bar``, or press it multiple
+   times. Each time it will select one line and move to
+   the next one. Each selected line will appear on the
+   right side.
+3. When all the required lines are displayed on the right
+   side, press ``Enter``. This will go back to the Python
+   REPL and show these lines as the current input. They
+   can still be edited from there.
+
+Key bindings
+------------
+
+Many Emacs and Vi navigation key bindings should work.
+Press ``F4`` to switch between Emacs and Vi mode.
+
+Additional bindings:
+
+- ``Space``: Select or delect a line.
+- ``Tab``: Move the focus between the history and input
+  pane. (Alternative: ``Ctrl-W``)
+- ``Ctrl-C``: Cancel. Ignore the result and go back to
+  the REPL. (Alternatives: ``q`` and ``Control-G``.)
+- ``Enter``: Accept the result and go back to the REPL.
+- ``F1``: Show/hide help. Press ``Enter`` to quit this
+  help message.
+
+Further, remember that searching works like in Emacs
+(using ``Ctrl-R``) or Vi (using ``/``).
+"""
+
+
+class BORDER:
+    " Box drawing characters. "
+    HORIZONTAL = "\u2501"
+    VERTICAL = "\u2503"
+    TOP_LEFT = "\u250f"
+    TOP_RIGHT = "\u2513"
+    BOTTOM_LEFT = "\u2517"
+    BOTTOM_RIGHT = "\u251b"
+    LIGHT_VERTICAL = "\u2502"
+
+
+def _create_popup_window(title: str, body: Container) -> Frame:
+    """
+    Return the layout for a pop-up window. It consists of a title bar showing
+    the `title` text, and a body layout. The window is surrounded by borders.
+    """
+    return Frame(body=body, title=title)
+
+
+class HistoryLayout:
+    """
+    Create and return a `Container` instance for the history
+    application.
+    """
+
+    def __init__(self, history):
+        search_toolbar = SearchToolbar()
+
+        self.help_buffer_control = BufferControl(
+            buffer=history.help_buffer, lexer=PygmentsLexer(RstLexer)
+        )
+
+        help_window = _create_popup_window(
+            title="History Help",
+            body=Window(
+                content=self.help_buffer_control,
+                right_margins=[ScrollbarMargin(display_arrows=True)],
+                scroll_offsets=ScrollOffsets(top=2, bottom=2),
+            ),
+        )
+
+        self.default_buffer_control = BufferControl(
+            buffer=history.default_buffer,
+            input_processors=[GrayExistingText(history.history_mapping)],
+            lexer=PygmentsLexer(PythonLexer),
+        )
+
+        self.history_buffer_control = BufferControl(
+            buffer=history.history_buffer,
+            lexer=PygmentsLexer(PythonLexer),
+            search_buffer_control=search_toolbar.control,
+            preview_search=True,
+        )
+
+        history_window = Window(
+            content=self.history_buffer_control,
+            wrap_lines=False,
+            left_margins=[HistoryMargin(history)],
+            scroll_offsets=ScrollOffsets(top=2, bottom=2),
+        )
+
+        self.root_container = HSplit(
+            [
+                #  Top title bar.
+                Window(
+                    content=FormattedTextControl(_get_top_toolbar_fragments),
+                    align=WindowAlign.CENTER,
+                    style="class:status-toolbar",
+                ),
+                FloatContainer(
+                    content=VSplit(
+                        [
+                            # Left side: history.
+                            history_window,
+                            # Separator.
+                            Window(
+                                width=D.exact(1),
+                                char=BORDER.LIGHT_VERTICAL,
+                                style="class:separator",
+                            ),
+                            # Right side: result.
+                            Window(
+                                content=self.default_buffer_control,
+                                wrap_lines=False,
+                                left_margins=[ResultMargin(history)],
+                                scroll_offsets=ScrollOffsets(top=2, bottom=2),
+                            ),
+                        ]
+                    ),
+                    floats=[
+                        # Help text as a float.
+                        Float(
+                            width=60,
+                            top=3,
+                            bottom=2,
+                            content=ConditionalContainer(
+                                content=help_window,
+                                filter=has_focus(history.help_buffer),
+                            ),
+                        )
+                    ],
+                ),
+                # Bottom toolbars.
+                ArgToolbar(),
+                search_toolbar,
+                Window(
+                    content=FormattedTextControl(
+                        partial(_get_bottom_toolbar_fragments, history=history)
+                    ),
+                    style="class:status-toolbar",
+                ),
+            ]
+        )
+
+        self.layout = Layout(self.root_container, history_window)
+
+
+def _get_top_toolbar_fragments():
+    return [("class:status-bar.title", "History browser - Insert from history")]
+
+
+def _get_bottom_toolbar_fragments(history):
+    python_input = history.python_input
+
+    @if_mousedown
+    def f1(mouse_event):
+        _toggle_help(history)
+
+    @if_mousedown
+    def tab(mouse_event):
+        _select_other_window(history)
+
+    return (
+        [("class:status-toolbar", " ")]
+        + get_inputmode_fragments(python_input)
+        + [
+            ("class:status-toolbar", " "),
+            ("class:status-toolbar.key", "[Space]"),
+            ("class:status-toolbar", " Toggle "),
+            ("class:status-toolbar.key", "[Tab]", tab),
+            ("class:status-toolbar", " Focus ", tab),
+            ("class:status-toolbar.key", "[Enter]"),
+            ("class:status-toolbar", " Accept "),
+            ("class:status-toolbar.key", "[F1]", f1),
+            ("class:status-toolbar", " Help ", f1),
+        ]
+    )
+
+
+class HistoryMargin(Margin):
+    """
+    Margin for the history buffer.
+    This displays a green bar for the selected entries.
+    """
+
+    def __init__(self, history):
+        self.history_buffer = history.history_buffer
+        self.history_mapping = history.history_mapping
+
+    def get_width(self, ui_content):
+        return 2
+
+    def create_margin(self, window_render_info, width, height):
+        document = self.history_buffer.document
+
+        lines_starting_new_entries = self.history_mapping.lines_starting_new_entries
+        selected_lines = self.history_mapping.selected_lines
+
+        current_lineno = document.cursor_position_row
+
+        visible_line_to_input_line = window_render_info.visible_line_to_input_line
+        result = []
+
+        for y in range(height):
+            line_number = visible_line_to_input_line.get(y)
+
+            # Show stars at the start of each entry.
+            # (Visualises multiline entries.)
+            if line_number in lines_starting_new_entries:
+                char = "*"
+            else:
+                char = " "
+
+            if line_number in selected_lines:
+                t = "class:history-line,selected"
+            else:
+                t = "class:history-line"
+
+            if line_number == current_lineno:
+                t = t + ",current"
+
+            result.append((t, char))
+            result.append(("", "\n"))
+
+        return result
+
+
+class ResultMargin(Margin):
+    """
+    The margin to be shown in the result pane.
+    """
+
+    def __init__(self, history):
+        self.history_mapping = history.history_mapping
+        self.history_buffer = history.history_buffer
+
+    def get_width(self, ui_content):
+        return 2
+
+    def create_margin(self, window_render_info, width, height):
+        document = self.history_buffer.document
+
+        current_lineno = document.cursor_position_row
+        offset = (
+            self.history_mapping.result_line_offset
+        )  # original_document.cursor_position_row
+
+        visible_line_to_input_line = window_render_info.visible_line_to_input_line
+
+        result = []
+
+        for y in range(height):
+            line_number = visible_line_to_input_line.get(y)
+
+            if (
+                line_number is None
+                or line_number < offset
+                or line_number >= offset + len(self.history_mapping.selected_lines)
+            ):
+                t = ""
+            elif line_number == current_lineno:
+                t = "class:history-line,selected,current"
+            else:
+                t = "class:history-line,selected"
+
+            result.append((t, " "))
+            result.append(("", "\n"))
+
+        return result
+
+    def invalidation_hash(self, document):
+        return document.cursor_position_row
+
+
+class GrayExistingText(Processor):
+    """
+    Turn the existing input, before and after the inserted code gray.
+    """
+
+    def __init__(self, history_mapping):
+        self.history_mapping = history_mapping
+        self._lines_before = len(
+            history_mapping.original_document.text_before_cursor.splitlines()
+        )
+
+    def apply_transformation(self, transformation_input):
+        lineno = transformation_input.lineno
+        fragments = transformation_input.fragments
+
+        if lineno < self._lines_before or lineno >= self._lines_before + len(
+            self.history_mapping.selected_lines
+        ):
+            text = fragment_list_to_text(fragments)
+            return Transformation(fragments=[("class:history.existing-input", text)])
+        else:
+            return Transformation(fragments=fragments)
+
+
+class HistoryMapping:
+    """
+    Keep a list of all the lines from the history and the selected lines.
+    """
+
+    def __init__(self, history, python_history, original_document):
+        self.history = history
+        self.python_history = python_history
+        self.original_document = original_document
+
+        self.lines_starting_new_entries = set()
+        self.selected_lines = set()
+
+        # Process history.
+        history_strings = python_history.get_strings()
+        history_lines = []
+
+        for entry_nr, entry in list(enumerate(history_strings))[-HISTORY_COUNT:]:
+            self.lines_starting_new_entries.add(len(history_lines))
+
+            for line in entry.splitlines():
+                history_lines.append(line)
+
+        if len(history_strings) > HISTORY_COUNT:
+            history_lines[0] = (
+                "# *** History has been truncated to %s lines ***" % HISTORY_COUNT
+            )
+
+        self.history_lines = history_lines
+        self.concatenated_history = "\n".join(history_lines)
+
+        # Line offset.
+        if self.original_document.text_before_cursor:
+            self.result_line_offset = self.original_document.cursor_position_row + 1
+        else:
+            self.result_line_offset = 0
+
+    def get_new_document(self, cursor_pos=None):
+        """
+        Create a `Document` instance that contains the resulting text.
+        """
+        lines = []
+
+        # Original text, before cursor.
+        if self.original_document.text_before_cursor:
+            lines.append(self.original_document.text_before_cursor)
+
+        # Selected entries from the history.
+        for line_no in sorted(self.selected_lines):
+            lines.append(self.history_lines[line_no])
+
+        # Original text, after cursor.
+        if self.original_document.text_after_cursor:
+            lines.append(self.original_document.text_after_cursor)
+
+        # Create `Document` with cursor at the right position.
+        text = "\n".join(lines)
+        if cursor_pos is not None and cursor_pos > len(text):
+            cursor_pos = len(text)
+        return Document(text, cursor_pos)
+
+    def update_default_buffer(self):
+        b = self.history.default_buffer
+
+        b.set_document(self.get_new_document(b.cursor_position), bypass_readonly=True)
+
+
+def _toggle_help(history):
+    " Display/hide help. "
+    help_buffer_control = history.history_layout.help_buffer_control
+
+    if history.app.layout.current_control == help_buffer_control:
+        history.app.layout.focus_previous()
+    else:
+        history.app.layout.current_control = help_buffer_control
+
+
+def _select_other_window(history):
+    " Toggle focus between left/right window. "
+    current_buffer = history.app.current_buffer
+    layout = history.history_layout.layout
+
+    if current_buffer == history.history_buffer:
+        layout.current_control = history.history_layout.default_buffer_control
+
+    elif current_buffer == history.default_buffer:
+        layout.current_control = history.history_layout.history_buffer_control
+
+
+def create_key_bindings(history, python_input, history_mapping):
+    """
+    Key bindings.
+    """
+    bindings = KeyBindings()
+    handle = bindings.add
+
+    @handle(" ", filter=has_focus(history.history_buffer))
+    def _(event):
+        """
+        Space: select/deselect line from history pane.
+        """
+        b = event.current_buffer
+        line_no = b.document.cursor_position_row
+
+        if not history_mapping.history_lines:
+            # If we've no history, then nothing to do
+            return
+
+        if line_no in history_mapping.selected_lines:
+            # Remove line.
+            history_mapping.selected_lines.remove(line_no)
+            history_mapping.update_default_buffer()
+        else:
+            # Add line.
+            history_mapping.selected_lines.add(line_no)
+            history_mapping.update_default_buffer()
+
+            # Update cursor position
+            default_buffer = history.default_buffer
+            default_lineno = (
+                sorted(history_mapping.selected_lines).index(line_no)
+                + history_mapping.result_line_offset
+            )
+            default_buffer.cursor_position = (
+                default_buffer.document.translate_row_col_to_index(default_lineno, 0)
+            )
+
+        # Also move the cursor to the next line. (This way they can hold
+        # space to select a region.)
+        b.cursor_position = b.document.translate_row_col_to_index(line_no + 1, 0)
+
+    @handle(" ", filter=has_focus(DEFAULT_BUFFER))
+    @handle("delete", filter=has_focus(DEFAULT_BUFFER))
+    @handle("c-h", filter=has_focus(DEFAULT_BUFFER))
+    def _(event):
+        """
+        Space: remove line from default pane.
+        """
+        b = event.current_buffer
+        line_no = b.document.cursor_position_row - history_mapping.result_line_offset
+
+        if line_no >= 0:
+            try:
+                history_lineno = sorted(history_mapping.selected_lines)[line_no]
+            except IndexError:
+                pass  # When `selected_lines` is an empty set.
+            else:
+                history_mapping.selected_lines.remove(history_lineno)
+
+            history_mapping.update_default_buffer()
+
+    help_focussed = has_focus(history.help_buffer)
+    main_buffer_focussed = has_focus(history.history_buffer) | has_focus(
+        history.default_buffer
+    )
+
+    @handle("tab", filter=main_buffer_focussed)
+    @handle("c-x", filter=main_buffer_focussed, eager=True)
+    # Eager: ignore the Emacs [Ctrl-X Ctrl-X] binding.
+    @handle("c-w", filter=main_buffer_focussed)
+    def _(event):
+        " Select other window. "
+        _select_other_window(history)
+
+    @handle("f4")
+    def _(event):
+        " Switch between Emacs/Vi mode. "
+        python_input.vi_mode = not python_input.vi_mode
+
+    @handle("f1")
+    def _(event):
+        " Display/hide help. "
+        _toggle_help(history)
+
+    @handle("enter", filter=help_focussed)
+    @handle("c-c", filter=help_focussed)
+    @handle("c-g", filter=help_focussed)
+    @handle("escape", filter=help_focussed)
+    def _(event):
+        " Leave help. "
+        event.app.layout.focus_previous()
+
+    @handle("q", filter=main_buffer_focussed)
+    @handle("f3", filter=main_buffer_focussed)
+    @handle("c-c", filter=main_buffer_focussed)
+    @handle("c-g", filter=main_buffer_focussed)
+    def _(event):
+        " Cancel and go back. "
+        event.app.exit(result=None)
+
+    @handle("enter", filter=main_buffer_focussed)
+    def _(event):
+        " Accept input. "
+        event.app.exit(result=history.default_buffer.text)
+
+    enable_system_bindings = Condition(lambda: python_input.enable_system_bindings)
+
+    @handle("c-z", filter=enable_system_bindings)
+    def _(event):
+        " Suspend to background. "
+        event.app.suspend_to_background()
+
+    return bindings
+
+
+class PythonHistory:
+    def __init__(self, python_input, original_document):
+        """
+        Create an `Application` for the history screen.
+        This has to be run as a sub application of `python_input`.
+
+        When this application runs and returns, it retuns the selected lines.
+        """
+        self.python_input = python_input
+
+        history_mapping = HistoryMapping(self, python_input.history, original_document)
+        self.history_mapping = history_mapping
+
+        document = Document(history_mapping.concatenated_history)
+        document = Document(
+            document.text,
+            cursor_position=document.cursor_position
+            + document.get_start_of_line_position(),
+        )
+
+        self.history_buffer = Buffer(
+            document=document,
+            on_cursor_position_changed=self._history_buffer_pos_changed,
+            accept_handler=(
+                lambda buff: get_app().exit(result=self.default_buffer.text)
+            ),
+            read_only=True,
+        )
+
+        self.default_buffer = Buffer(
+            name=DEFAULT_BUFFER,
+            document=history_mapping.get_new_document(),
+            on_cursor_position_changed=self._default_buffer_pos_changed,
+            read_only=True,
+        )
+
+        self.help_buffer = Buffer(document=Document(HELP_TEXT, 0), read_only=True)
+
+        self.history_layout = HistoryLayout(self)
+
+        self.app = Application(
+            layout=self.history_layout.layout,
+            full_screen=True,
+            style=python_input._current_style,
+            mouse_support=Condition(lambda: python_input.enable_mouse_support),
+            key_bindings=create_key_bindings(self, python_input, history_mapping),
+        )
+
+    def _default_buffer_pos_changed(self, _):
+        """When the cursor changes in the default buffer. Synchronize with
+        history buffer."""
+        # Only when this buffer has the focus.
+        if self.app.current_buffer == self.default_buffer:
+            try:
+                line_no = (
+                    self.default_buffer.document.cursor_position_row
+                    - self.history_mapping.result_line_offset
+                )
+
+                if line_no < 0:  # When the cursor is above the inserted region.
+                    raise IndexError
+
+                history_lineno = sorted(self.history_mapping.selected_lines)[line_no]
+            except IndexError:
+                pass
+            else:
+                self.history_buffer.cursor_position = (
+                    self.history_buffer.document.translate_row_col_to_index(
+                        history_lineno, 0
+                    )
+                )
+
+    def _history_buffer_pos_changed(self, _):
+        """ When the cursor changes in the history buffer. Synchronize. """
+        # Only when this buffer has the focus.
+        if self.app.current_buffer == self.history_buffer:
+            line_no = self.history_buffer.document.cursor_position_row
+
+            if line_no in self.history_mapping.selected_lines:
+                default_lineno = (
+                    sorted(self.history_mapping.selected_lines).index(line_no)
+                    + self.history_mapping.result_line_offset
+                )
+
+                self.default_buffer.cursor_position = (
+                    self.default_buffer.document.translate_row_col_to_index(
+                        default_lineno, 0
+                    )
+                )

ptpython/ipython.py

@@ -0,0 +1,285 @@
+"""
+
+Adaptor for using the input system of `prompt_toolkit` with the IPython
+backend.
+
+This gives a powerful interactive shell that has a nice user interface, but
+also the power of for instance all the %-magic functions that IPython has to
+offer.
+
+"""
+from warnings import warn
+
+from IPython import utils as ipy_utils
+from IPython.core.inputsplitter import IPythonInputSplitter
+from IPython.terminal.embed import InteractiveShellEmbed as _InteractiveShellEmbed
+from IPython.terminal.ipapp import load_default_config
+from prompt_toolkit.completion import (
+    Completer,
+    Completion,
+    PathCompleter,
+    WordCompleter,
+)
+from prompt_toolkit.contrib.completers import SystemCompleter
+from prompt_toolkit.contrib.regular_languages.compiler import compile
+from prompt_toolkit.contrib.regular_languages.completion import GrammarCompleter
+from prompt_toolkit.contrib.regular_languages.lexer import GrammarLexer
+from prompt_toolkit.document import Document
+from prompt_toolkit.formatted_text import PygmentsTokens
+from prompt_toolkit.lexers import PygmentsLexer, SimpleLexer
+from prompt_toolkit.styles import Style
+from pygments.lexers import BashLexer, PythonLexer
+
+from ptpython.prompt_style import PromptStyle
+
+from .python_input import PythonCompleter, PythonInput, PythonValidator
+from .style import default_ui_style
+
+__all__ = ["embed"]
+
+
+class IPythonPrompt(PromptStyle):
+    """
+    Style for IPython >5.0, use the prompt_toolkit tokens directly.
+    """
+
+    def __init__(self, prompts):
+        self.prompts = prompts
+
+    def in_prompt(self):
+        return PygmentsTokens(self.prompts.in_prompt_tokens())
+
+    def in2_prompt(self, width):
+        return PygmentsTokens(self.prompts.continuation_prompt_tokens())
+
+    def out_prompt(self):
+        return []
+
+
+class IPythonValidator(PythonValidator):
+    def __init__(self, *args, **kwargs):
+        super(IPythonValidator, self).__init__(*args, **kwargs)
+        self.isp = IPythonInputSplitter()
+
+    def validate(self, document):
+        document = Document(text=self.isp.transform_cell(document.text))
+        super(IPythonValidator, self).validate(document)
+
+
+def create_ipython_grammar():
+    """
+    Return compiled IPython grammar.
+    """
+    return compile(
+        r"""
+        \s*
+        (
+            (?P<percent>%)(
+                (?P<magic>pycat|run|loadpy|load)  \s+ (?P<py_filename>[^\s]+)  |
+                (?P<magic>cat)                    \s+ (?P<filename>[^\s]+)     |
+                (?P<magic>pushd|cd|ls)            \s+ (?P<directory>[^\s]+)    |
+                (?P<magic>pdb)                    \s+ (?P<pdb_arg>[^\s]+)      |
+                (?P<magic>autocall)               \s+ (?P<autocall_arg>[^\s]+) |
+                (?P<magic>time|timeit|prun)       \s+ (?P<python>.+)           |
+                (?P<magic>psource|pfile|pinfo|pinfo2) \s+ (?P<python>.+)       |
+                (?P<magic>system)                 \s+ (?P<system>.+)           |
+                (?P<magic>unalias)                \s+ (?P<alias_name>.+)       |
+                (?P<magic>[^\s]+)   .* |
+            ) .*            |
+            !(?P<system>.+) |
+            (?![%!]) (?P<python>.+)
+        )
+        \s*
+    """
+    )
+
+
+def create_completer(
+    get_globals,
+    get_locals,
+    magics_manager,
+    alias_manager,
+    get_enable_dictionary_completion,
+):
+    g = create_ipython_grammar()
+
+    return GrammarCompleter(
+        g,
+        {
+            "python": PythonCompleter(
+                get_globals, get_locals, get_enable_dictionary_completion
+            ),
+            "magic": MagicsCompleter(magics_manager),
+            "alias_name": AliasCompleter(alias_manager),
+            "pdb_arg": WordCompleter(["on", "off"], ignore_case=True),
+            "autocall_arg": WordCompleter(["0", "1", "2"], ignore_case=True),
+            "py_filename": PathCompleter(
+                only_directories=False, file_filter=lambda name: name.endswith(".py")
+            ),
+            "filename": PathCompleter(only_directories=False),
+            "directory": PathCompleter(only_directories=True),
+            "system": SystemCompleter(),
+        },
+    )
+
+
+def create_lexer():
+    g = create_ipython_grammar()
+
+    return GrammarLexer(
+        g,
+        lexers={
+            "percent": SimpleLexer("class:pygments.operator"),
+            "magic": SimpleLexer("class:pygments.keyword"),
+            "filename": SimpleLexer("class:pygments.name"),
+            "python": PygmentsLexer(PythonLexer),
+            "system": PygmentsLexer(BashLexer),
+        },
+    )
+
+
+class MagicsCompleter(Completer):
+    def __init__(self, magics_manager):
+        self.magics_manager = magics_manager
+
+    def get_completions(self, document, complete_event):
+        text = document.text_before_cursor.lstrip()
+
+        for m in sorted(self.magics_manager.magics["line"]):
+            if m.startswith(text):
+                yield Completion("%s" % m, -len(text))
+
+
+class AliasCompleter(Completer):
+    def __init__(self, alias_manager):
+        self.alias_manager = alias_manager
+
+    def get_completions(self, document, complete_event):
+        text = document.text_before_cursor.lstrip()
+        # aliases = [a for a, _ in self.alias_manager.aliases]
+        aliases = self.alias_manager.aliases
+
+        for a, cmd in sorted(aliases, key=lambda a: a[0]):
+            if a.startswith(text):
+                yield Completion("%s" % a, -len(text), display_meta=cmd)
+
+
+class IPythonInput(PythonInput):
+    """
+    Override our `PythonCommandLineInterface` to add IPython specific stuff.
+    """
+
+    def __init__(self, ipython_shell, *a, **kw):
+        kw["_completer"] = create_completer(
+            kw["get_globals"],
+            kw["get_globals"],
+            ipython_shell.magics_manager,
+            ipython_shell.alias_manager,
+            lambda: self.enable_dictionary_completion,
+        )
+        kw["_lexer"] = create_lexer()
+        kw["_validator"] = IPythonValidator(get_compiler_flags=self.get_compiler_flags)
+
+        super().__init__(*a, **kw)
+        self.ipython_shell = ipython_shell
+
+        self.all_prompt_styles["ipython"] = IPythonPrompt(ipython_shell.prompts)
+        self.prompt_style = "ipython"
+
+        # UI style for IPython. Add tokens that are used by IPython>5.0
+        style_dict = {}
+        style_dict.update(default_ui_style)
+        style_dict.update(
+            {
+                "pygments.prompt": "#009900",
+                "pygments.prompt-num": "#00ff00 bold",
+                "pygments.out-prompt": "#990000",
+                "pygments.out-prompt-num": "#ff0000 bold",
+            }
+        )
+
+        self.ui_styles = {"default": Style.from_dict(style_dict)}
+        self.use_ui_colorscheme("default")
+
+
+class InteractiveShellEmbed(_InteractiveShellEmbed):
+    """
+    Override the `InteractiveShellEmbed` from IPython, to replace the front-end
+    with our input shell.
+
+    :param configure: Callable for configuring the repl.
+    """
+
+    def __init__(self, *a, **kw):
+        vi_mode = kw.pop("vi_mode", False)
+        history_filename = kw.pop("history_filename", None)
+        configure = kw.pop("configure", None)
+        title = kw.pop("title", None)
+
+        # Don't ask IPython to confirm for exit. We have our own exit prompt.
+        self.confirm_exit = False
+
+        super().__init__(*a, **kw)
+
+        def get_globals():
+            return self.user_ns
+
+        python_input = IPythonInput(
+            self,
+            get_globals=get_globals,
+            vi_mode=vi_mode,
+            history_filename=history_filename,
+        )
+
+        if title:
+            python_input.terminal_title = title
+
+        if configure:
+            configure(python_input)
+            python_input.prompt_style = "ipython"  # Don't take from config.
+
+        self.python_input = python_input
+
+    def prompt_for_code(self):
+        try:
+            return self.python_input.app.run()
+        except KeyboardInterrupt:
+            self.python_input.default_buffer.document = Document()
+            return ""
+
+
+def initialize_extensions(shell, extensions):
+    """
+    Partial copy of `InteractiveShellApp.init_extensions` from IPython.
+    """
+    try:
+        iter(extensions)
+    except TypeError:
+        pass  # no extensions found
+    else:
+        for ext in extensions:
+            try:
+                shell.extension_manager.load_extension(ext)
+            except:
+                warn(
+                    "Error in loading extension: %s" % ext
+                    + "\nCheck your config files in %s"
+                    % ipy_utils.path.get_ipython_dir()
+                )
+                shell.showtraceback()
+
+
+def embed(**kwargs):
+    """
+    Copied from `IPython/terminal/embed.py`, but using our `InteractiveShellEmbed` instead.
+    """
+    config = kwargs.get("config")
+    header = kwargs.pop("header", "")
+    compile_flags = kwargs.pop("compile_flags", None)
+    if config is None:
+        config = load_default_config()
+        config.InteractiveShellEmbed = config.TerminalInteractiveShell
+        kwargs["config"] = config
+    shell = InteractiveShellEmbed.instance(**kwargs)
+    initialize_extensions(shell, config["InteractiveShellApp"]["extensions"])
+    shell(header=header, stack_depth=2, compile_flags=compile_flags)

ptpython/key_bindings.py

@@ -0,0 +1,326 @@
+from prompt_toolkit.application import get_app
+from prompt_toolkit.document import Document
+from prompt_toolkit.enums import DEFAULT_BUFFER
+from prompt_toolkit.filters import (
+    Condition,
+    emacs_insert_mode,
+    emacs_mode,
+    has_focus,
+    has_selection,
+    vi_insert_mode,
+)
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.key_binding.bindings.named_commands import get_by_name
+from prompt_toolkit.keys import Keys
+
+from .utils import document_is_multiline_python
+
+__all__ = [
+    "load_python_bindings",
+    "load_sidebar_bindings",
+    "load_confirm_exit_bindings",
+]
+
+
+@Condition
+def tab_should_insert_whitespace():
+    """
+    When the 'tab' key is pressed with only whitespace character before the
+    cursor, do autocompletion. Otherwise, insert indentation.
+
+    Except for the first character at the first line. Then always do a
+    completion. It doesn't make sense to start the first line with
+    indentation.
+    """
+    b = get_app().current_buffer
+    before_cursor = b.document.current_line_before_cursor
+
+    return bool(b.text and (not before_cursor or before_cursor.isspace()))
+
+
+def load_python_bindings(python_input):
+    """
+    Custom key bindings.
+    """
+    bindings = KeyBindings()
+
+    sidebar_visible = Condition(lambda: python_input.show_sidebar)
+    handle = bindings.add
+
+    @handle("c-l")
+    def _(event):
+        """
+        Clear whole screen and render again -- also when the sidebar is visible.
+        """
+        event.app.renderer.clear()
+
+    @handle("c-z")
+    def _(event):
+        """
+        Suspend.
+        """
+        if python_input.enable_system_bindings:
+            event.app.suspend_to_background()
+
+    # Delete word before cursor, but use all Python symbols as separators
+    # (WORD=False).
+    handle("c-w")(get_by_name("backward-kill-word"))
+
+    @handle("f2")
+    def _(event):
+        """
+        Show/hide sidebar.
+        """
+        python_input.show_sidebar = not python_input.show_sidebar
+        if python_input.show_sidebar:
+            event.app.layout.focus(python_input.ptpython_layout.sidebar)
+        else:
+            event.app.layout.focus_last()
+
+    @handle("f3")
+    def _(event):
+        """
+        Select from the history.
+        """
+        python_input.enter_history()
+
+    @handle("f4")
+    def _(event):
+        """
+        Toggle between Vi and Emacs mode.
+        """
+        python_input.vi_mode = not python_input.vi_mode
+
+    @handle("f6")
+    def _(event):
+        """
+        Enable/Disable paste mode.
+        """
+        python_input.paste_mode = not python_input.paste_mode
+
+    @handle(
+        "tab", filter=~sidebar_visible & ~has_selection & tab_should_insert_whitespace
+    )
+    def _(event):
+        """
+        When tab should insert whitespace, do that instead of completion.
+        """
+        event.app.current_buffer.insert_text("    ")
+
+    @Condition
+    def is_multiline():
+        return document_is_multiline_python(python_input.default_buffer.document)
+
+    @handle(
+        "enter",
+        filter=~sidebar_visible
+        & ~has_selection
+        & (vi_insert_mode | emacs_insert_mode)
+        & has_focus(DEFAULT_BUFFER)
+        & ~is_multiline,
+    )
+    @handle(Keys.Escape, Keys.Enter, filter=~sidebar_visible & emacs_mode)
+    def _(event):
+        """
+        Accept input (for single line input).
+        """
+        b = event.current_buffer
+
+        if b.validate():
+            # When the cursor is at the end, and we have an empty line:
+            # drop the empty lines, but return the value.
+            b.document = Document(
+                text=b.text.rstrip(), cursor_position=len(b.text.rstrip())
+            )
+
+            b.validate_and_handle()
+
+    @handle(
+        "enter",
+        filter=~sidebar_visible
+        & ~has_selection
+        & (vi_insert_mode | emacs_insert_mode)
+        & has_focus(DEFAULT_BUFFER)
+        & is_multiline,
+    )
+    def _(event):
+        """
+        Behaviour of the Enter key.
+
+        Auto indent after newline/Enter.
+        (When not in Vi navigaton mode, and when multiline is enabled.)
+        """
+        b = event.current_buffer
+        empty_lines_required = python_input.accept_input_on_enter or 10000
+
+        def at_the_end(b):
+            """we consider the cursor at the end when there is no text after
+            the cursor, or only whitespace."""
+            text = b.document.text_after_cursor
+            return text == "" or (text.isspace() and not "\n" in text)
+
+        if python_input.paste_mode:
+            # In paste mode, always insert text.
+            b.insert_text("\n")
+
+        elif at_the_end(b) and b.document.text.replace(" ", "").endswith(
+            "\n" * (empty_lines_required - 1)
+        ):
+            # When the cursor is at the end, and we have an empty line:
+            # drop the empty lines, but return the value.
+            if b.validate():
+                b.document = Document(
+                    text=b.text.rstrip(), cursor_position=len(b.text.rstrip())
+                )
+
+                b.validate_and_handle()
+        else:
+            auto_newline(b)
+
+    @handle(
+        "c-d",
+        filter=~sidebar_visible
+        & has_focus(python_input.default_buffer)
+        & Condition(
+            lambda:
+            # The current buffer is empty.
+            not get_app().current_buffer.text
+        ),
+    )
+    def _(event):
+        """
+        Override Control-D exit, to ask for confirmation.
+        """
+        if python_input.confirm_exit:
+            # Show exit confirmation and focus it (focusing is important for
+            # making sure the default buffer key bindings are not active).
+            python_input.show_exit_confirmation = True
+            python_input.app.layout.focus(
+                python_input.ptpython_layout.exit_confirmation
+            )
+        else:
+            event.app.exit(exception=EOFError)
+
+    @handle("c-c", filter=has_focus(python_input.default_buffer))
+    def _(event):
+        " Abort when Control-C has been pressed. "
+        event.app.exit(exception=KeyboardInterrupt, style="class:aborting")
+
+    return bindings
+
+
+def load_sidebar_bindings(python_input):
+    """
+    Load bindings for the navigation in the sidebar.
+    """
+    bindings = KeyBindings()
+
+    handle = bindings.add
+    sidebar_visible = Condition(lambda: python_input.show_sidebar)
+
+    @handle("up", filter=sidebar_visible)
+    @handle("c-p", filter=sidebar_visible)
+    @handle("k", filter=sidebar_visible)
+    def _(event):
+        " Go to previous option. "
+        python_input.selected_option_index = (
+            python_input.selected_option_index - 1
+        ) % python_input.option_count
+
+    @handle("down", filter=sidebar_visible)
+    @handle("c-n", filter=sidebar_visible)
+    @handle("j", filter=sidebar_visible)
+    def _(event):
+        " Go to next option. "
+        python_input.selected_option_index = (
+            python_input.selected_option_index + 1
+        ) % python_input.option_count
+
+    @handle("right", filter=sidebar_visible)
+    @handle("l", filter=sidebar_visible)
+    @handle(" ", filter=sidebar_visible)
+    def _(event):
+        " Select next value for current option. "
+        option = python_input.selected_option
+        option.activate_next()
+
+    @handle("left", filter=sidebar_visible)
+    @handle("h", filter=sidebar_visible)
+    def _(event):
+        " Select previous value for current option. "
+        option = python_input.selected_option
+        option.activate_previous()
+
+    @handle("c-c", filter=sidebar_visible)
+    @handle("c-d", filter=sidebar_visible)
+    @handle("c-d", filter=sidebar_visible)
+    @handle("enter", filter=sidebar_visible)
+    @handle("escape", filter=sidebar_visible)
+    def _(event):
+        " Hide sidebar. "
+        python_input.show_sidebar = False
+        event.app.layout.focus_last()
+
+    return bindings
+
+
+def load_confirm_exit_bindings(python_input):
+    """
+    Handle yes/no key presses when the exit confirmation is shown.
+    """
+    bindings = KeyBindings()
+
+    handle = bindings.add
+    confirmation_visible = Condition(lambda: python_input.show_exit_confirmation)
+
+    @handle("y", filter=confirmation_visible)
+    @handle("Y", filter=confirmation_visible)
+    @handle("enter", filter=confirmation_visible)
+    @handle("c-d", filter=confirmation_visible)
+    def _(event):
+        """
+        Really quit.
+        """
+        event.app.exit(exception=EOFError, style="class:exiting")
+
+    @handle(Keys.Any, filter=confirmation_visible)
+    def _(event):
+        """
+        Cancel exit.
+        """
+        python_input.show_exit_confirmation = False
+        python_input.app.layout.focus_previous()
+
+    return bindings
+
+
+def auto_newline(buffer):
+    r"""
+    Insert \n at the cursor position. Also add necessary padding.
+    """
+    insert_text = buffer.insert_text
+
+    if buffer.document.current_line_after_cursor:
+        # When we are in the middle of a line. Always insert a newline.
+        insert_text("\n")
+    else:
+        # Go to new line, but also add indentation.
+        current_line = buffer.document.current_line_before_cursor.rstrip()
+        insert_text("\n")
+
+        # Unident if the last line ends with 'pass', remove four spaces.
+        unindent = current_line.rstrip().endswith(" pass")
+
+        # Copy whitespace from current line
+        current_line2 = current_line[4:] if unindent else current_line
+
+        for c in current_line2:
+            if c.isspace():
+                insert_text(c)
+            else:
+                break
+
+        # If the last line ends with a colon, add four extra spaces.
+        if current_line[-1:] == ":":
+            for x in range(4):
+                insert_text(" ")

ptpython/layout.py

@@ -0,0 +1,763 @@
+"""
+Creation of the `Layout` instance for the Python input/REPL.
+"""
+import platform
+import sys
+from enum import Enum
+from inspect import _ParameterKind as ParameterKind
+from typing import TYPE_CHECKING, Optional
+
+from prompt_toolkit.application import get_app
+from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
+from prompt_toolkit.filters import (
+    Condition,
+    has_focus,
+    is_done,
+    renderer_height_is_known,
+)
+from prompt_toolkit.formatted_text import fragment_list_width, to_formatted_text
+from prompt_toolkit.formatted_text.base import StyleAndTextTuples
+from prompt_toolkit.key_binding.vi_state import InputMode
+from prompt_toolkit.layout.containers import (
+    ConditionalContainer,
+    Container,
+    Float,
+    FloatContainer,
+    HSplit,
+    ScrollOffsets,
+    VSplit,
+    Window,
+)
+from prompt_toolkit.layout.controls import BufferControl, FormattedTextControl
+from prompt_toolkit.layout.dimension import AnyDimension, Dimension
+from prompt_toolkit.layout.layout import Layout
+from prompt_toolkit.layout.margins import PromptMargin
+from prompt_toolkit.layout.menus import CompletionsMenu, MultiColumnCompletionsMenu
+from prompt_toolkit.layout.processors import (
+    AppendAutoSuggestion,
+    ConditionalProcessor,
+    DisplayMultipleCursors,
+    HighlightIncrementalSearchProcessor,
+    HighlightMatchingBracketProcessor,
+    HighlightSelectionProcessor,
+    TabsProcessor,
+)
+from prompt_toolkit.lexers import SimpleLexer
+from prompt_toolkit.mouse_events import MouseEvent
+from prompt_toolkit.selection import SelectionType
+from prompt_toolkit.widgets.toolbars import (
+    ArgToolbar,
+    CompletionsToolbar,
+    SearchToolbar,
+    SystemToolbar,
+    ValidationToolbar,
+)
+from pygments.lexers import PythonLexer
+
+from .filters import HasSignature, ShowDocstring, ShowSidebar, ShowSignature
+from .utils import if_mousedown
+
+if TYPE_CHECKING:
+    from .python_input import OptionCategory, PythonInput
+
+__all__ = ["PtPythonLayout", "CompletionVisualisation"]
+
+
+class CompletionVisualisation(Enum):
+    " Visualisation method for the completions. "
+    NONE = "none"
+    POP_UP = "pop-up"
+    MULTI_COLUMN = "multi-column"
+    TOOLBAR = "toolbar"
+
+
+def show_completions_toolbar(python_input: "PythonInput") -> Condition:
+    return Condition(
+        lambda: python_input.completion_visualisation == CompletionVisualisation.TOOLBAR
+    )
+
+
+def show_completions_menu(python_input: "PythonInput") -> Condition:
+    return Condition(
+        lambda: python_input.completion_visualisation == CompletionVisualisation.POP_UP
+    )
+
+
+def show_multi_column_completions_menu(python_input: "PythonInput") -> Condition:
+    return Condition(
+        lambda: python_input.completion_visualisation
+        == CompletionVisualisation.MULTI_COLUMN
+    )
+
+
+def python_sidebar(python_input: "PythonInput") -> Window:
+    """
+    Create the `Layout` for the sidebar with the configurable options.
+    """
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        tokens: StyleAndTextTuples = []
+
+        def append_category(category: "OptionCategory") -> None:
+            tokens.extend(
+                [
+                    ("class:sidebar", "  "),
+                    ("class:sidebar.title", "   %-36s" % category.title),
+                    ("class:sidebar", "\n"),
+                ]
+            )
+
+        def append(index: int, label: str, status: str) -> None:
+            selected = index == python_input.selected_option_index
+
+            @if_mousedown
+            def select_item(mouse_event: MouseEvent) -> None:
+                python_input.selected_option_index = index
+
+            @if_mousedown
+            def goto_next(mouse_event: MouseEvent) -> None:
+                " Select item and go to next value. "
+                python_input.selected_option_index = index
+                option = python_input.selected_option
+                option.activate_next()
+
+            sel = ",selected" if selected else ""
+
+            tokens.append(("class:sidebar" + sel, " >" if selected else "  "))
+            tokens.append(("class:sidebar.label" + sel, "%-24s" % label, select_item))
+            tokens.append(("class:sidebar.status" + sel, " ", select_item))
+            tokens.append(("class:sidebar.status" + sel, "%s" % status, goto_next))
+
+            if selected:
+                tokens.append(("[SetCursorPosition]", ""))
+
+            tokens.append(
+                ("class:sidebar.status" + sel, " " * (13 - len(status)), goto_next)
+            )
+            tokens.append(("class:sidebar", "<" if selected else ""))
+            tokens.append(("class:sidebar", "\n"))
+
+        i = 0
+        for category in python_input.options:
+            append_category(category)
+
+            for option in category.options:
+                append(i, option.title, "%s" % option.get_current_value())
+                i += 1
+
+        tokens.pop()  # Remove last newline.
+
+        return tokens
+
+    class Control(FormattedTextControl):
+        def move_cursor_down(self):
+            python_input.selected_option_index += 1
+
+        def move_cursor_up(self):
+            python_input.selected_option_index -= 1
+
+    return Window(
+        Control(get_text_fragments),
+        style="class:sidebar",
+        width=Dimension.exact(43),
+        height=Dimension(min=3),
+        scroll_offsets=ScrollOffsets(top=1, bottom=1),
+    )
+
+
+def python_sidebar_navigation(python_input):
+    """
+    Create the `Layout` showing the navigation information for the sidebar.
+    """
+
+    def get_text_fragments():
+        # Show navigation info.
+        return [
+            ("class:sidebar", "    "),
+            ("class:sidebar.key", "[Arrows]"),
+            ("class:sidebar", " "),
+            ("class:sidebar.description", "Navigate"),
+            ("class:sidebar", " "),
+            ("class:sidebar.key", "[Enter]"),
+            ("class:sidebar", " "),
+            ("class:sidebar.description", "Hide menu"),
+        ]
+
+    return Window(
+        FormattedTextControl(get_text_fragments),
+        style="class:sidebar",
+        width=Dimension.exact(43),
+        height=Dimension.exact(1),
+    )
+
+
+def python_sidebar_help(python_input):
+    """
+    Create the `Layout` for the help text for the current item in the sidebar.
+    """
+    token = "class:sidebar.helptext"
+
+    def get_current_description():
+        """
+        Return the description of the selected option.
+        """
+        i = 0
+        for category in python_input.options:
+            for option in category.options:
+                if i == python_input.selected_option_index:
+                    return option.description
+                i += 1
+        return ""
+
+    def get_help_text():
+        return [(token, get_current_description())]
+
+    return ConditionalContainer(
+        content=Window(
+            FormattedTextControl(get_help_text),
+            style=token,
+            height=Dimension(min=3),
+            wrap_lines=True,
+        ),
+        filter=ShowSidebar(python_input)
+        & Condition(lambda: python_input.show_sidebar_help)
+        & ~is_done,
+    )
+
+
+def signature_toolbar(python_input):
+    """
+    Return the `Layout` for the signature.
+    """
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        result: StyleAndTextTuples = []
+        append = result.append
+        Signature = "class:signature-toolbar"
+
+        if python_input.signatures:
+            sig = python_input.signatures[0]  # Always take the first one.
+
+            append((Signature, " "))
+            try:
+                append((Signature, sig.name))
+            except IndexError:
+                # Workaround for #37: https://github.com/jonathanslenders/python-prompt-toolkit/issues/37
+                # See also: https://github.com/davidhalter/jedi/issues/490
+                return []
+
+            append((Signature + ",operator", "("))
+
+            got_positional_only = False
+            got_keyword_only = False
+
+            for i, p in enumerate(sig.parameters):
+                # Detect transition between positional-only and not positional-only.
+                if p.kind == ParameterKind.POSITIONAL_ONLY:
+                    got_positional_only = True
+                if got_positional_only and p.kind != ParameterKind.POSITIONAL_ONLY:
+                    got_positional_only = False
+                    append((Signature, "/"))
+                    append((Signature + ",operator", ", "))
+
+                if not got_keyword_only and p.kind == ParameterKind.KEYWORD_ONLY:
+                    got_keyword_only = True
+                    append((Signature, "*"))
+                    append((Signature + ",operator", ", "))
+
+                sig_index = getattr(sig, "index", 0)
+
+                if i == sig_index:
+                    # Note: we use `_Param.description` instead of
+                    #       `_Param.name`, that way we also get the '*' before args.
+                    append((Signature + ",current-name", p.description))
+                else:
+                    append((Signature, p.description))
+
+                if p.default:
+                    # NOTE: For the jedi-based completion, the default is
+                    #       currently still part of the name.
+                    append((Signature, f"={p.default}"))
+
+                append((Signature + ",operator", ", "))
+
+            if sig.parameters:
+                # Pop last comma
+                result.pop()
+
+            append((Signature + ",operator", ")"))
+            append((Signature, " "))
+        return result
+
+    return ConditionalContainer(
+        content=Window(
+            FormattedTextControl(get_text_fragments), height=Dimension.exact(1)
+        ),
+        filter=
+        # Show only when there is a signature
+        HasSignature(python_input) &
+        # Signature needs to be shown.
+        ShowSignature(python_input) &
+        # And no sidebar is visible.
+        ~ShowSidebar(python_input) &
+        # Not done yet.
+        ~is_done,
+    )
+
+
+class PythonPromptMargin(PromptMargin):
+    """
+    Create margin that displays the prompt.
+    It shows something like "In [1]:".
+    """
+
+    def __init__(self, python_input) -> None:
+        self.python_input = python_input
+
+        def get_prompt_style():
+            return python_input.all_prompt_styles[python_input.prompt_style]
+
+        def get_prompt() -> StyleAndTextTuples:
+            return to_formatted_text(get_prompt_style().in_prompt())
+
+        def get_continuation(width, line_number, is_soft_wrap):
+            if python_input.show_line_numbers and not is_soft_wrap:
+                text = ("%i " % (line_number + 1)).rjust(width)
+                return [("class:line-number", text)]
+            else:
+                return get_prompt_style().in2_prompt(width)
+
+        super().__init__(get_prompt, get_continuation)
+
+
+def status_bar(python_input: "PythonInput") -> Container:
+    """
+    Create the `Layout` for the status bar.
+    """
+    TB = "class:status-toolbar"
+
+    @if_mousedown
+    def toggle_paste_mode(mouse_event: MouseEvent) -> None:
+        python_input.paste_mode = not python_input.paste_mode
+
+    @if_mousedown
+    def enter_history(mouse_event: MouseEvent) -> None:
+        python_input.enter_history()
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        python_buffer = python_input.default_buffer
+
+        result: StyleAndTextTuples = []
+        append = result.append
+
+        append((TB, " "))
+        result.extend(get_inputmode_fragments(python_input))
+        append((TB, " "))
+
+        # Position in history.
+        append(
+            (
+                TB,
+                "%i/%i "
+                % (python_buffer.working_index + 1, len(python_buffer._working_lines)),
+            )
+        )
+
+        # Shortcuts.
+        app = get_app()
+        if (
+            not python_input.vi_mode
+            and app.current_buffer == python_input.search_buffer
+        ):
+            append((TB, "[Ctrl-G] Cancel search [Enter] Go to this position."))
+        elif bool(app.current_buffer.selection_state) and not python_input.vi_mode:
+            # Emacs cut/copy keys.
+            append((TB, "[Ctrl-W] Cut [Meta-W] Copy [Ctrl-Y] Paste [Ctrl-G] Cancel"))
+        else:
+            result.extend(
+                [
+                    (TB + " class:status-toolbar.key", "[F3]", enter_history),
+                    (TB, " History ", enter_history),
+                    (TB + " class:status-toolbar.key", "[F6]", toggle_paste_mode),
+                    (TB, " ", toggle_paste_mode),
+                ]
+            )
+
+            if python_input.paste_mode:
+                append(
+                    (TB + " class:paste-mode-on", "Paste mode (on)", toggle_paste_mode)
+                )
+            else:
+                append((TB, "Paste mode", toggle_paste_mode))
+
+        return result
+
+    return ConditionalContainer(
+        content=Window(content=FormattedTextControl(get_text_fragments), style=TB),
+        filter=~is_done
+        & renderer_height_is_known
+        & Condition(
+            lambda: python_input.show_status_bar
+            and not python_input.show_exit_confirmation
+        ),
+    )
+
+
+def get_inputmode_fragments(python_input: "PythonInput") -> StyleAndTextTuples:
+    """
+    Return current input mode as a list of (token, text) tuples for use in a
+    toolbar.
+    """
+    app = get_app()
+
+    @if_mousedown
+    def toggle_vi_mode(mouse_event: MouseEvent) -> None:
+        python_input.vi_mode = not python_input.vi_mode
+
+    token = "class:status-toolbar"
+    input_mode_t = "class:status-toolbar.input-mode"
+
+    mode = app.vi_state.input_mode
+    result: StyleAndTextTuples = []
+    append = result.append
+
+    if python_input.title:
+        result.extend(to_formatted_text(python_input.title))
+
+    append((input_mode_t, "[F4] ", toggle_vi_mode))
+
+    # InputMode
+    if python_input.vi_mode:
+        recording_register = app.vi_state.recording_register
+        if recording_register:
+            append((token, " "))
+            append((token + " class:record", "RECORD({})".format(recording_register)))
+            append((token, " - "))
+
+        if app.current_buffer.selection_state is not None:
+            if app.current_buffer.selection_state.type == SelectionType.LINES:
+                append((input_mode_t, "Vi (VISUAL LINE)", toggle_vi_mode))
+            elif app.current_buffer.selection_state.type == SelectionType.CHARACTERS:
+                append((input_mode_t, "Vi (VISUAL)", toggle_vi_mode))
+                append((token, " "))
+            elif app.current_buffer.selection_state.type == SelectionType.BLOCK:
+                append((input_mode_t, "Vi (VISUAL BLOCK)", toggle_vi_mode))
+                append((token, " "))
+        elif mode in (InputMode.INSERT, "vi-insert-multiple"):
+            append((input_mode_t, "Vi (INSERT)", toggle_vi_mode))
+            append((token, "  "))
+        elif mode == InputMode.NAVIGATION:
+            append((input_mode_t, "Vi (NAV)", toggle_vi_mode))
+            append((token, "     "))
+        elif mode == InputMode.REPLACE:
+            append((input_mode_t, "Vi (REPLACE)", toggle_vi_mode))
+            append((token, " "))
+    else:
+        if app.emacs_state.is_recording:
+            append((token, " "))
+            append((token + " class:record", "RECORD"))
+            append((token, " - "))
+
+        append((input_mode_t, "Emacs", toggle_vi_mode))
+        append((token, " "))
+
+    return result
+
+
+def show_sidebar_button_info(python_input: "PythonInput") -> Container:
+    """
+    Create `Layout` for the information in the right-bottom corner.
+    (The right part of the status bar.)
+    """
+
+    @if_mousedown
+    def toggle_sidebar(mouse_event: MouseEvent) -> None:
+        " Click handler for the menu. "
+        python_input.show_sidebar = not python_input.show_sidebar
+
+    version = sys.version_info
+    tokens: StyleAndTextTuples = [
+        ("class:status-toolbar.key", "[F2]", toggle_sidebar),
+        ("class:status-toolbar", " Menu", toggle_sidebar),
+        ("class:status-toolbar", " - "),
+        (
+            "class:status-toolbar.python-version",
+            "%s %i.%i.%i"
+            % (platform.python_implementation(), version[0], version[1], version[2]),
+        ),
+        ("class:status-toolbar", " "),
+    ]
+    width = fragment_list_width(tokens)
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        # Python version
+        return tokens
+
+    return ConditionalContainer(
+        content=Window(
+            FormattedTextControl(get_text_fragments),
+            style="class:status-toolbar",
+            height=Dimension.exact(1),
+            width=Dimension.exact(width),
+        ),
+        filter=~is_done
+        & renderer_height_is_known
+        & Condition(
+            lambda: python_input.show_status_bar
+            and not python_input.show_exit_confirmation
+        ),
+    )
+
+
+def create_exit_confirmation(
+    python_input: "PythonInput", style="class:exit-confirmation"
+) -> Container:
+    """
+    Create `Layout` for the exit message.
+    """
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        # Show "Do you really want to exit?"
+        return [
+            (style, "\n %s ([y]/n) " % python_input.exit_message),
+            ("[SetCursorPosition]", ""),
+            (style, "  \n"),
+        ]
+
+    visible = ~is_done & Condition(lambda: python_input.show_exit_confirmation)
+
+    return ConditionalContainer(
+        content=Window(
+            FormattedTextControl(get_text_fragments, focusable=True), style=style
+        ),
+        filter=visible,
+    )
+
+
+def meta_enter_message(python_input: "PythonInput") -> Container:
+    """
+    Create the `Layout` for the 'Meta+Enter` message.
+    """
+
+    def get_text_fragments() -> StyleAndTextTuples:
+        return [("class:accept-message", " [Meta+Enter] Execute ")]
+
+    @Condition
+    def extra_condition() -> bool:
+        " Only show when... "
+        b = python_input.default_buffer
+
+        return (
+            python_input.show_meta_enter_message
+            and (
+                not b.document.is_cursor_at_the_end
+                or python_input.accept_input_on_enter is None
+            )
+            and "\n" in b.text
+        )
+
+    visible = ~is_done & has_focus(DEFAULT_BUFFER) & extra_condition
+
+    return ConditionalContainer(
+        content=Window(FormattedTextControl(get_text_fragments)), filter=visible
+    )
+
+
+class PtPythonLayout:
+    def __init__(
+        self,
+        python_input: "PythonInput",
+        lexer=PythonLexer,
+        extra_body=None,
+        extra_toolbars=None,
+        extra_buffer_processors=None,
+        input_buffer_height: Optional[AnyDimension] = None,
+    ) -> None:
+        D = Dimension
+        extra_body = [extra_body] if extra_body else []
+        extra_toolbars = extra_toolbars or []
+        extra_buffer_processors = extra_buffer_processors or []
+        input_buffer_height = input_buffer_height or D(min=6)
+
+        search_toolbar = SearchToolbar(python_input.search_buffer)
+
+        def create_python_input_window():
+            def menu_position():
+                """
+                When there is no autocompletion menu to be shown, and we have a
+                signature, set the pop-up position at `bracket_start`.
+                """
+                b = python_input.default_buffer
+
+                if python_input.signatures:
+                    row, col = python_input.signatures[0].bracket_start
+                    index = b.document.translate_row_col_to_index(row - 1, col)
+                    return index
+
+            return Window(
+                BufferControl(
+                    buffer=python_input.default_buffer,
+                    search_buffer_control=search_toolbar.control,
+                    lexer=lexer,
+                    include_default_input_processors=False,
+                    input_processors=[
+                        ConditionalProcessor(
+                            processor=HighlightIncrementalSearchProcessor(),
+                            filter=has_focus(SEARCH_BUFFER)
+                            | has_focus(search_toolbar.control),
+                        ),
+                        HighlightSelectionProcessor(),
+                        DisplayMultipleCursors(),
+                        TabsProcessor(),
+                        # Show matching parentheses, but only while editing.
+                        ConditionalProcessor(
+                            processor=HighlightMatchingBracketProcessor(chars="[](){}"),
+                            filter=has_focus(DEFAULT_BUFFER)
+                            & ~is_done
+                            & Condition(
+                                lambda: python_input.highlight_matching_parenthesis
+                            ),
+                        ),
+                        ConditionalProcessor(
+                            processor=AppendAutoSuggestion(), filter=~is_done
+                        ),
+                    ]
+                    + extra_buffer_processors,
+                    menu_position=menu_position,
+                    # Make sure that we always see the result of an reverse-i-search:
+                    preview_search=True,
+                ),
+                left_margins=[PythonPromptMargin(python_input)],
+                # Scroll offsets. The 1 at the bottom is important to make sure
+                # the cursor is never below the "Press [Meta+Enter]" message
+                # which is a float.
+                scroll_offsets=ScrollOffsets(bottom=1, left=4, right=4),
+                # As long as we're editing, prefer a minimal height of 6.
+                height=(
+                    lambda: (
+                        None
+                        if get_app().is_done or python_input.show_exit_confirmation
+                        else input_buffer_height
+                    )
+                ),
+                wrap_lines=Condition(lambda: python_input.wrap_lines),
+            )
+
+        sidebar = python_sidebar(python_input)
+        self.exit_confirmation = create_exit_confirmation(python_input)
+
+        root_container = HSplit(
+            [
+                VSplit(
+                    [
+                        HSplit(
+                            [
+                                FloatContainer(
+                                    content=HSplit(
+                                        [create_python_input_window()] + extra_body
+                                    ),
+                                    floats=[
+                                        Float(
+                                            xcursor=True,
+                                            ycursor=True,
+                                            content=HSplit(
+                                                [
+                                                    signature_toolbar(python_input),
+                                                    ConditionalContainer(
+                                                        content=CompletionsMenu(
+                                                            scroll_offset=(
+                                                                lambda: python_input.completion_menu_scroll_offset
+                                                            ),
+                                                            max_height=12,
+                                                        ),
+                                                        filter=show_completions_menu(
+                                                            python_input
+                                                        ),
+                                                    ),
+                                                    ConditionalContainer(
+                                                        content=MultiColumnCompletionsMenu(),
+                                                        filter=show_multi_column_completions_menu(
+                                                            python_input
+                                                        ),
+                                                    ),
+                                                ]
+                                            ),
+                                        ),
+                                        Float(
+                                            left=2,
+                                            bottom=1,
+                                            content=self.exit_confirmation,
+                                        ),
+                                        Float(
+                                            bottom=0,
+                                            right=0,
+                                            height=1,
+                                            content=meta_enter_message(python_input),
+                                            hide_when_covering_content=True,
+                                        ),
+                                        Float(
+                                            bottom=1,
+                                            left=1,
+                                            right=0,
+                                            content=python_sidebar_help(python_input),
+                                        ),
+                                    ],
+                                ),
+                                ArgToolbar(),
+                                search_toolbar,
+                                SystemToolbar(),
+                                ValidationToolbar(),
+                                ConditionalContainer(
+                                    content=CompletionsToolbar(),
+                                    filter=show_completions_toolbar(python_input)
+                                    & ~is_done,
+                                ),
+                                # Docstring region.
+                                ConditionalContainer(
+                                    content=Window(
+                                        height=D.exact(1),
+                                        char="\u2500",
+                                        style="class:separator",
+                                    ),
+                                    filter=HasSignature(python_input)
+                                    & ShowDocstring(python_input)
+                                    & ~is_done,
+                                ),
+                                ConditionalContainer(
+                                    content=Window(
+                                        BufferControl(
+                                            buffer=python_input.docstring_buffer,
+                                            lexer=SimpleLexer(style="class:docstring"),
+                                            # lexer=PythonLexer,
+                                        ),
+                                        height=D(max=12),
+                                    ),
+                                    filter=HasSignature(python_input)
+                                    & ShowDocstring(python_input)
+                                    & ~is_done,
+                                ),
+                            ]
+                        ),
+                        ConditionalContainer(
+                            content=HSplit(
+                                [
+                                    sidebar,
+                                    Window(style="class:sidebar,separator", height=1),
+                                    python_sidebar_navigation(python_input),
+                                ]
+                            ),
+                            filter=ShowSidebar(python_input) & ~is_done,
+                        ),
+                    ]
+                )
+            ]
+            + extra_toolbars
+            + [
+                VSplit(
+                    [status_bar(python_input), show_sidebar_button_info(python_input)]
+                )
+            ]
+        )
+
+        self.layout = Layout(root_container)
+        self.sidebar = sidebar

ptpython/lexer.py

@@ -0,0 +1,28 @@
+from typing import Callable, Optional
+
+from prompt_toolkit.document import Document
+from prompt_toolkit.formatted_text import StyleAndTextTuples
+from prompt_toolkit.lexers import Lexer, PygmentsLexer
+from pygments.lexers import BashLexer
+from pygments.lexers import Python3Lexer as PythonLexer
+
+__all__ = ["PtpythonLexer"]
+
+
+class PtpythonLexer(Lexer):
+    """
+    Lexer for ptpython input.
+
+    If the input starts with an exclamation mark, use a Bash lexer, otherwise,
+    use a Python 3 lexer.
+    """
+
+    def __init__(self, python_lexer: Optional[Lexer] = None) -> None:
+        self.python_lexer = python_lexer or PygmentsLexer(PythonLexer)
+        self.system_lexer = PygmentsLexer(BashLexer)
+
+    def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]:
+        if document.text.startswith("!"):
+            return self.system_lexer.lex_document(document)
+
+        return self.python_lexer.lex_document(document)

ptpython/prompt_style.py

@@ -0,0 +1,77 @@
+from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING
+
+from prompt_toolkit.formatted_text import AnyFormattedText
+
+if TYPE_CHECKING:
+    from .python_input import PythonInput
+
+__all__ = ["PromptStyle", "IPythonPrompt", "ClassicPrompt"]
+
+
+class PromptStyle(metaclass=ABCMeta):
+    """
+    Base class for all prompts.
+    """
+
+    @abstractmethod
+    def in_prompt(self) -> AnyFormattedText:
+        " Return the input tokens. "
+        return []
+
+    @abstractmethod
+    def in2_prompt(self, width: int) -> AnyFormattedText:
+        """
+        Tokens for every following input line.
+
+        :param width: The available width. This is coming from the width taken
+                      by `in_prompt`.
+        """
+        return []
+
+    @abstractmethod
+    def out_prompt(self) -> AnyFormattedText:
+        " Return the output tokens. "
+        return []
+
+
+class IPythonPrompt(PromptStyle):
+    """
+    A prompt resembling the IPython prompt.
+    """
+
+    def __init__(self, python_input: "PythonInput") -> None:
+        self.python_input = python_input
+
+    def in_prompt(self) -> AnyFormattedText:
+        return [
+            ("class:in", "In ["),
+            ("class:in.number", "%s" % self.python_input.current_statement_index),
+            ("class:in", "]: "),
+        ]
+
+    def in2_prompt(self, width: int) -> AnyFormattedText:
+        return [("class:in", "...: ".rjust(width))]
+
+    def out_prompt(self) -> AnyFormattedText:
+        return [
+            ("class:out", "Out["),
+            ("class:out.number", "%s" % self.python_input.current_statement_index),
+            ("class:out", "]:"),
+            ("", " "),
+        ]
+
+
+class ClassicPrompt(PromptStyle):
+    """
+    The classic Python prompt.
+    """
+
+    def in_prompt(self) -> AnyFormattedText:
+        return [("class:prompt", ">>> ")]
+
+    def in2_prompt(self, width: int) -> AnyFormattedText:
+        return [("class:prompt.dots", "...")]
+
+    def out_prompt(self) -> AnyFormattedText:
+        return []

ptpython/repl.py

@@ -0,0 +1,765 @@
+"""
+Utility for creating a Python repl.
+
+::
+
+    from ptpython.repl import embed
+    embed(globals(), locals(), vi_mode=False)
+
+"""
+import asyncio
+import builtins
+import os
+import sys
+import threading
+import traceback
+import types
+import warnings
+from dis import COMPILER_FLAG_NAMES
+from enum import Enum
+from typing import Any, Callable, ContextManager, Dict, Optional
+
+from prompt_toolkit.formatted_text import (
+    HTML,
+    AnyFormattedText,
+    FormattedText,
+    PygmentsTokens,
+    StyleAndTextTuples,
+    fragment_list_width,
+    merge_formatted_text,
+    to_formatted_text,
+)
+from prompt_toolkit.formatted_text.utils import fragment_list_to_text, split_lines
+from prompt_toolkit.key_binding import KeyBindings, KeyPressEvent
+from prompt_toolkit.patch_stdout import patch_stdout as patch_stdout_context
+from prompt_toolkit.shortcuts import (
+    PromptSession,
+    clear_title,
+    print_formatted_text,
+    set_title,
+)
+from prompt_toolkit.styles import BaseStyle
+from prompt_toolkit.utils import DummyContext, get_cwidth
+from pygments.lexers import PythonLexer, PythonTracebackLexer
+from pygments.token import Token
+
+from .python_input import PythonInput
+
+try:
+    from ast import PyCF_ALLOW_TOP_LEVEL_AWAIT  # type: ignore
+except ImportError:
+    PyCF_ALLOW_TOP_LEVEL_AWAIT = 0
+
+__all__ = ["PythonRepl", "enable_deprecation_warnings", "run_config", "embed"]
+
+
+def _get_coroutine_flag() -> Optional[int]:
+    for k, v in COMPILER_FLAG_NAMES.items():
+        if v == "COROUTINE":
+            return k
+
+    # Flag not found.
+    return None
+
+
+COROUTINE_FLAG: Optional[int] = _get_coroutine_flag()
+
+
+def _has_coroutine_flag(code: types.CodeType) -> bool:
+    if COROUTINE_FLAG is None:
+        # Not supported on this Python version.
+        return False
+
+    return bool(code.co_flags & COROUTINE_FLAG)
+
+
+class PythonRepl(PythonInput):
+    def __init__(self, *a, **kw) -> None:
+        self._startup_paths = kw.pop("startup_paths", None)
+        super().__init__(*a, **kw)
+        self._load_start_paths()
+
+    def _load_start_paths(self) -> None:
+        " Start the Read-Eval-Print Loop. "
+        if self._startup_paths:
+            for path in self._startup_paths:
+                if os.path.exists(path):
+                    with open(path, "rb") as f:
+                        code = compile(f.read(), path, "exec")
+                        exec(code, self.get_globals(), self.get_locals())
+                else:
+                    output = self.app.output
+                    output.write("WARNING | File not found: {}\n\n".format(path))
+
+    def run(self) -> None:
+        """
+        Run the REPL loop.
+        """
+        if self.terminal_title:
+            set_title(self.terminal_title)
+
+        self._add_to_namespace()
+
+        try:
+            while True:
+                try:
+                    # Read.
+                    try:
+                        text = self.read()
+                    except EOFError:
+                        return
+
+                    # Eval.
+                    try:
+                        result = self.eval(text)
+                    except KeyboardInterrupt as e:  # KeyboardInterrupt doesn't inherit from Exception.
+                        raise
+                    except SystemExit:
+                        return
+                    except BaseException as e:
+                        self._handle_exception(e)
+                    else:
+                        # Print.
+                        if result is not None:
+                            self.show_result(result)
+
+                        # Loop.
+                        self.current_statement_index += 1
+                        self.signatures = []
+
+                except KeyboardInterrupt as e:
+                    # Handle all possible `KeyboardInterrupt` errors. This can
+                    # happen during the `eval`, but also during the
+                    # `show_result` if something takes too long.
+                    # (Try/catch is around the whole block, because we want to
+                    # prevent that a Control-C keypress terminates the REPL in
+                    # any case.)
+                    self._handle_keyboard_interrupt(e)
+        finally:
+            if self.terminal_title:
+                clear_title()
+            self._remove_from_namespace()
+
+    async def run_async(self) -> None:
+        """
+        Run the REPL loop, but run the blocking parts in an executor, so that
+        we don't block the event loop. Both the input and output (which can
+        display a pager) will run in a separate thread with their own event
+        loop, this way ptpython's own event loop won't interfere with the
+        asyncio event loop from where this is called.
+
+        The "eval" however happens in the current thread, which is important.
+        (Both for control-C to work, as well as for the code to see the right
+        thread in which it was embedded).
+        """
+        loop = asyncio.get_event_loop()
+
+        if self.terminal_title:
+            set_title(self.terminal_title)
+
+        self._add_to_namespace()
+
+        try:
+            while True:
+                try:
+                    # Read.
+                    try:
+                        text = await loop.run_in_executor(None, self.read)
+                    except EOFError:
+                        return
+
+                    # Eval.
+                    try:
+                        result = await self.eval_async(text)
+                    except KeyboardInterrupt as e:  # KeyboardInterrupt doesn't inherit from Exception.
+                        raise
+                    except SystemExit:
+                        return
+                    except BaseException as e:
+                        self._handle_exception(e)
+                    else:
+                        # Print.
+                        if result is not None:
+                            await loop.run_in_executor(
+                                None, lambda: self.show_result(result)
+                            )
+
+                        # Loop.
+                        self.current_statement_index += 1
+                        self.signatures = []
+
+                except KeyboardInterrupt as e:
+                    # XXX: This does not yet work properly. In some situations,
+                    # `KeyboardInterrupt` exceptions can end up in the event
+                    # loop selector.
+                    self._handle_keyboard_interrupt(e)
+        finally:
+            if self.terminal_title:
+                clear_title()
+            self._remove_from_namespace()
+
+    def eval(self, line: str) -> object:
+        """
+        Evaluate the line and print the result.
+        """
+        # WORKAROUND: Due to a bug in Jedi, the current directory is removed
+        # from sys.path. See: https://github.com/davidhalter/jedi/issues/1148
+        if "" not in sys.path:
+            sys.path.insert(0, "")
+
+        if line.lstrip().startswith("!"):
+            # Run as shell command
+            os.system(line[1:])
+        else:
+            # Try eval first
+            try:
+                code = self._compile_with_flags(line, "eval")
+            except SyntaxError:
+                pass
+            else:
+                # No syntax errors for eval. Do eval.
+                result = eval(code, self.get_globals(), self.get_locals())
+
+                if _has_coroutine_flag(code):
+                    result = asyncio.get_event_loop().run_until_complete(result)
+
+                self._store_eval_result(result)
+                return result
+
+            # If not a valid `eval` expression, run using `exec` instead.
+            # Note that we shouldn't run this in the `except SyntaxError` block
+            # above, then `sys.exc_info()` would not report the right error.
+            # See issue: https://github.com/prompt-toolkit/ptpython/issues/435
+            code = self._compile_with_flags(line, "exec")
+            exec(code, self.get_globals(), self.get_locals())
+
+        return None
+
+    async def eval_async(self, line: str) -> object:
+        """
+        Evaluate the line and print the result.
+        """
+        # WORKAROUND: Due to a bug in Jedi, the current directory is removed
+        # from sys.path. See: https://github.com/davidhalter/jedi/issues/1148
+        if "" not in sys.path:
+            sys.path.insert(0, "")
+
+        if line.lstrip().startswith("!"):
+            # Run as shell command
+            os.system(line[1:])
+        else:
+            # Try eval first
+            try:
+                code = self._compile_with_flags(line, "eval")
+            except SyntaxError:
+                pass
+            else:
+                # No syntax errors for eval. Do eval.
+                result = eval(code, self.get_globals(), self.get_locals())
+
+                if _has_coroutine_flag(code):
+                    result = await result
+
+                self._store_eval_result(result)
+                return result
+
+            # If not a valid `eval` expression, run using `exec` instead.
+            code = self._compile_with_flags(line, "exec")
+            exec(code, self.get_globals(), self.get_locals())
+
+        return None
+
+    def _store_eval_result(self, result: object) -> None:
+        locals: Dict[str, Any] = self.get_locals()
+        locals["_"] = locals["_%i" % self.current_statement_index] = result
+
+    def get_compiler_flags(self) -> int:
+        return super().get_compiler_flags() | PyCF_ALLOW_TOP_LEVEL_AWAIT
+
+    def _compile_with_flags(self, code: str, mode: str):
+        " Compile code with the right compiler flags. "
+        return compile(
+            code,
+            "<stdin>",
+            mode,
+            flags=self.get_compiler_flags(),
+            dont_inherit=True,
+        )
+
+    def show_result(self, result: object) -> None:
+        """
+        Show __repr__ for an `eval` result.
+
+        Note: this can raise `KeyboardInterrupt` if either calling `__repr__`,
+              `__pt_repr__` or formatting the output with "Black" takes to long
+              and the user presses Control-C.
+        """
+        out_prompt = to_formatted_text(self.get_output_prompt())
+
+        # If the repr is valid Python code, use the Pygments lexer.
+        try:
+            result_repr = repr(result)
+        except KeyboardInterrupt:
+            raise  # Don't catch here.
+        except BaseException as e:
+            # Calling repr failed.
+            self._handle_exception(e)
+            return
+
+        try:
+            compile(result_repr, "", "eval")
+        except SyntaxError:
+            formatted_result_repr = to_formatted_text(result_repr)
+        else:
+            # Syntactically correct. Format with black and syntax highlight.
+            if self.enable_output_formatting:
+                # Inline import. Slightly speed up start-up time if black is
+                # not used.
+                import black
+
+                result_repr = black.format_str(
+                    result_repr,
+                    mode=black.FileMode(line_length=self.app.output.get_size().columns),
+                )
+
+            formatted_result_repr = to_formatted_text(
+                PygmentsTokens(list(_lex_python_result(result_repr)))
+            )
+
+        # If __pt_repr__ is present, take this. This can return prompt_toolkit
+        # formatted text.
+        try:
+            if hasattr(result, "__pt_repr__"):
+                formatted_result_repr = to_formatted_text(
+                    getattr(result, "__pt_repr__")()
+                )
+                if isinstance(formatted_result_repr, list):
+                    formatted_result_repr = FormattedText(formatted_result_repr)
+        except KeyboardInterrupt:
+            raise  # Don't catch here.
+        except:
+            # For bad code, `__getattr__` can raise something that's not an
+            # `AttributeError`. This happens already when calling `hasattr()`.
+            pass
+
+        # Align every line to the prompt.
+        line_sep = "\n" + " " * fragment_list_width(out_prompt)
+        indented_repr: StyleAndTextTuples = []
+
+        lines = list(split_lines(formatted_result_repr))
+
+        for i, fragment in enumerate(lines):
+            indented_repr.extend(fragment)
+
+            # Add indentation separator between lines, not after the last line.
+            if i != len(lines) - 1:
+                indented_repr.append(("", line_sep))
+
+        # Write output tokens.
+        if self.enable_syntax_highlighting:
+            formatted_output = merge_formatted_text([out_prompt, indented_repr])
+        else:
+            formatted_output = FormattedText(
+                out_prompt + [("", fragment_list_to_text(formatted_result_repr))]
+            )
+
+        if self.enable_pager:
+            self.print_paginated_formatted_text(to_formatted_text(formatted_output))
+        else:
+            self.print_formatted_text(to_formatted_text(formatted_output))
+
+        self.app.output.flush()
+
+        if self.insert_blank_line_after_output:
+            self.app.output.write("\n")
+
+    def print_formatted_text(
+        self, formatted_text: StyleAndTextTuples, end: str = "\n"
+    ) -> None:
+        print_formatted_text(
+            FormattedText(formatted_text),
+            style=self._current_style,
+            style_transformation=self.style_transformation,
+            include_default_pygments_style=False,
+            output=self.app.output,
+            end=end,
+        )
+
+    def print_paginated_formatted_text(
+        self,
+        formatted_text: StyleAndTextTuples,
+        end: str = "\n",
+    ) -> None:
+        """
+        Print formatted text, using --MORE-- style pagination.
+        (Avoid filling up the terminal's scrollback buffer.)
+        """
+        pager_prompt = self.create_pager_prompt()
+        size = self.app.output.get_size()
+
+        abort = False
+        print_all = False
+
+        # Max number of lines allowed in the buffer before painting.
+        max_rows = size.rows - 1
+
+        # Page buffer.
+        rows_in_buffer = 0
+        columns_in_buffer = 0
+        page: StyleAndTextTuples = []
+
+        def flush_page() -> None:
+            nonlocal page, columns_in_buffer, rows_in_buffer
+            self.print_formatted_text(page, end="")
+            page = []
+            columns_in_buffer = 0
+            rows_in_buffer = 0
+
+        def show_pager() -> None:
+            nonlocal abort, max_rows, print_all
+
+            # Run pager prompt in another thread.
+            # Same as for the input. This prevents issues with nested event
+            # loops.
+            pager_result = None
+
+            def in_thread() -> None:
+                nonlocal pager_result
+                pager_result = pager_prompt.prompt()
+
+            th = threading.Thread(target=in_thread)
+            th.start()
+            th.join()
+
+            if pager_result == PagerResult.ABORT:
+                print("...")
+                abort = True
+
+            elif pager_result == PagerResult.NEXT_LINE:
+                max_rows = 1
+
+            elif pager_result == PagerResult.NEXT_PAGE:
+                max_rows = size.rows - 1
+
+            elif pager_result == PagerResult.PRINT_ALL:
+                print_all = True
+
+        # Loop over lines. Show --MORE-- prompt when page is filled.
+
+        formatted_text = formatted_text + [("", end)]
+        lines = list(split_lines(formatted_text))
+
+        for lineno, line in enumerate(lines):
+            for style, text, *_ in line:
+                for c in text:
+                    width = get_cwidth(c)
+
+                    # (Soft) wrap line if it doesn't fit.
+                    if columns_in_buffer + width > size.columns:
+                        # Show pager first if we get too many lines after
+                        # wrapping.
+                        if rows_in_buffer + 1 >= max_rows and not print_all:
+                            page.append(("", "\n"))
+                            flush_page()
+                            show_pager()
+                            if abort:
+                                return
+
+                        rows_in_buffer += 1
+                        columns_in_buffer = 0
+
+                    columns_in_buffer += width
+                    page.append((style, c))
+
+            if rows_in_buffer + 1 >= max_rows and not print_all:
+                page.append(("", "\n"))
+                flush_page()
+                show_pager()
+                if abort:
+                    return
+            else:
+                # Add line ending between lines (if `end="\n"` was given, one
+                # more empty line is added in `split_lines` automatically to
+                # take care of the final line ending).
+                if lineno != len(lines) - 1:
+                    page.append(("", "\n"))
+                    rows_in_buffer += 1
+                    columns_in_buffer = 0
+
+        flush_page()
+
+    def create_pager_prompt(self) -> PromptSession["PagerResult"]:
+        """
+        Create pager --MORE-- prompt.
+        """
+        return create_pager_prompt(self._current_style, self.title)
+
+    def _handle_exception(self, e: BaseException) -> None:
+        output = self.app.output
+
+        # Instead of just calling ``traceback.format_exc``, we take the
+        # traceback and skip the bottom calls of this framework.
+        t, v, tb = sys.exc_info()
+
+        # Required for pdb.post_mortem() to work.
+        sys.last_type, sys.last_value, sys.last_traceback = t, v, tb
+
+        tblist = list(traceback.extract_tb(tb))
+
+        for line_nr, tb_tuple in enumerate(tblist):
+            if tb_tuple[0] == "<stdin>":
+                tblist = tblist[line_nr:]
+                break
+
+        l = traceback.format_list(tblist)
+        if l:
+            l.insert(0, "Traceback (most recent call last):\n")
+        l.extend(traceback.format_exception_only(t, v))
+
+        tb_str = "".join(l)
+
+        # Format exception and write to output.
+        # (We use the default style. Most other styles result
+        # in unreadable colors for the traceback.)
+        if self.enable_syntax_highlighting:
+            tokens = list(_lex_python_traceback(tb_str))
+        else:
+            tokens = [(Token, tb_str)]
+
+        print_formatted_text(
+            PygmentsTokens(tokens),
+            style=self._current_style,
+            style_transformation=self.style_transformation,
+            include_default_pygments_style=False,
+            output=output,
+        )
+
+        output.write("%s\n" % e)
+        output.flush()
+
+    def _handle_keyboard_interrupt(self, e: KeyboardInterrupt) -> None:
+        output = self.app.output
+
+        output.write("\rKeyboardInterrupt\n\n")
+        output.flush()
+
+    def _add_to_namespace(self) -> None:
+        """
+        Add ptpython built-ins to global namespace.
+        """
+        globals = self.get_globals()
+
+        # Add a 'get_ptpython', similar to 'get_ipython'
+        def get_ptpython() -> PythonInput:
+            return self
+
+        globals["get_ptpython"] = get_ptpython
+
+    def _remove_from_namespace(self) -> None:
+        """
+        Remove added symbols from the globals.
+        """
+        globals = self.get_globals()
+        del globals["get_ptpython"]
+
+
+def _lex_python_traceback(tb):
+    " Return token list for traceback string. "
+    lexer = PythonTracebackLexer()
+    return lexer.get_tokens(tb)
+
+
+def _lex_python_result(tb):
+    " Return token list for Python string. "
+    lexer = PythonLexer()
+    # Use `get_tokens_unprocessed`, so that we get exactly the same string,
+    # without line endings appended. `print_formatted_text` already appends a
+    # line ending, and otherwise we'll have two line endings.
+    tokens = lexer.get_tokens_unprocessed(tb)
+    return [(tokentype, value) for index, tokentype, value in tokens]
+
+
+def enable_deprecation_warnings() -> None:
+    """
+    Show deprecation warnings, when they are triggered directly by actions in
+    the REPL. This is recommended to call, before calling `embed`.
+
+    e.g. This will show an error message when the user imports the 'sha'
+         library on Python 2.7.
+    """
+    warnings.filterwarnings("default", category=DeprecationWarning, module="__main__")
+
+
+def run_config(repl: PythonInput, config_file: str = "~/.ptpython/config.py") -> None:
+    """
+    Execute REPL config file.
+
+    :param repl: `PythonInput` instance.
+    :param config_file: Path of the configuration file.
+    """
+    # Expand tildes.
+    config_file = os.path.expanduser(config_file)
+
+    def enter_to_continue() -> None:
+        input("\nPress ENTER to continue...")
+
+    # Check whether this file exists.
+    if not os.path.exists(config_file):
+        print("Impossible to read %r" % config_file)
+        enter_to_continue()
+        return
+
+    # Run the config file in an empty namespace.
+    try:
+        namespace: Dict[str, Any] = {}
+
+        with open(config_file, "rb") as f:
+            code = compile(f.read(), config_file, "exec")
+            exec(code, namespace, namespace)
+
+        # Now we should have a 'configure' method in this namespace. We call this
+        # method with the repl as an argument.
+        if "configure" in namespace:
+            namespace["configure"](repl)
+
+    except Exception:
+        traceback.print_exc()
+        enter_to_continue()
+
+
+def embed(
+    globals=None,
+    locals=None,
+    configure: Optional[Callable[[PythonRepl], None]] = None,
+    vi_mode: bool = False,
+    history_filename: Optional[str] = None,
+    title: Optional[str] = None,
+    startup_paths=None,
+    patch_stdout: bool = False,
+    return_asyncio_coroutine: bool = False,
+) -> None:
+    """
+    Call this to embed  Python shell at the current point in your program.
+    It's similar to `IPython.embed` and `bpython.embed`. ::
+
+        from prompt_toolkit.contrib.repl import embed
+        embed(globals(), locals())
+
+    :param vi_mode: Boolean. Use Vi instead of Emacs key bindings.
+    :param configure: Callable that will be called with the `PythonRepl` as a first
+                      argument, to trigger configuration.
+    :param title: Title to be displayed in the terminal titlebar. (None or string.)
+    :param patch_stdout:  When true, patch `sys.stdout` so that background
+        threads that are printing will print nicely above the prompt.
+    """
+    # Default globals/locals
+    if globals is None:
+        globals = {
+            "__name__": "__main__",
+            "__package__": None,
+            "__doc__": None,
+            "__builtins__": builtins,
+        }
+
+    locals = locals or globals
+
+    def get_globals():
+        return globals
+
+    def get_locals():
+        return locals
+
+    # Create REPL.
+    repl = PythonRepl(
+        get_globals=get_globals,
+        get_locals=get_locals,
+        vi_mode=vi_mode,
+        history_filename=history_filename,
+        startup_paths=startup_paths,
+    )
+
+    if title:
+        repl.terminal_title = title
+
+    if configure:
+        configure(repl)
+
+    # Start repl.
+    patch_context: ContextManager = (
+        patch_stdout_context() if patch_stdout else DummyContext()
+    )
+
+    if return_asyncio_coroutine:
+
+        async def coroutine():
+            with patch_context:
+                await repl.run_async()
+
+        return coroutine()
+    else:
+        with patch_context:
+            repl.run()
+
+
+class PagerResult(Enum):
+    ABORT = "ABORT"
+    NEXT_LINE = "NEXT_LINE"
+    NEXT_PAGE = "NEXT_PAGE"
+    PRINT_ALL = "PRINT_ALL"
+
+
+def create_pager_prompt(
+    style: BaseStyle, title: AnyFormattedText = ""
+) -> PromptSession[PagerResult]:
+    """
+    Create a "continue" prompt for paginated output.
+    """
+    bindings = KeyBindings()
+
+    @bindings.add("enter")
+    @bindings.add("down")
+    def next_line(event: KeyPressEvent) -> None:
+        event.app.exit(result=PagerResult.NEXT_LINE)
+
+    @bindings.add("space")
+    def next_page(event: KeyPressEvent) -> None:
+        event.app.exit(result=PagerResult.NEXT_PAGE)
+
+    @bindings.add("a")
+    def print_all(event: KeyPressEvent) -> None:
+        event.app.exit(result=PagerResult.PRINT_ALL)
+
+    @bindings.add("q")
+    @bindings.add("c-c")
+    @bindings.add("c-d")
+    @bindings.add("escape", eager=True)
+    def no(event: KeyPressEvent) -> None:
+        event.app.exit(result=PagerResult.ABORT)
+
+    @bindings.add("<any>")
+    def _(event: KeyPressEvent) -> None:
+        " Disallow inserting other text. "
+        pass
+
+    style
+
+    session: PromptSession[PagerResult] = PromptSession(
+        merge_formatted_text(
+            [
+                title,
+                HTML(
+                    "<status-toolbar>"
+                    "<more> -- MORE -- </more> "
+                    "<key>[Enter]</key> Scroll "
+                    "<key>[Space]</key> Next page "
+                    "<key>[a]</key> Print all "
+                    "<key>[q]</key> Quit "
+                    "</status-toolbar>: "
+                ),
+            ]
+        ),
+        key_bindings=bindings,
+        erase_when_done=True,
+        style=style,
+    )
+    return session

ptpython/signatures.py

@@ -0,0 +1,266 @@
+"""
+Helpers for retrieving the function signature of the function call that we are
+editing.
+
+Either with the Jedi library, or using `inspect.signature` if Jedi fails and we
+can use `eval()` to evaluate the function object.
+"""
+import inspect
+from inspect import Signature as InspectSignature
+from inspect import _ParameterKind as ParameterKind
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from prompt_toolkit.document import Document
+
+from .completer import DictionaryCompleter
+from .utils import get_jedi_script_from_document
+
+__all__ = ["Signature", "get_signatures_using_jedi", "get_signatures_using_eval"]
+
+
+class Parameter:
+    def __init__(
+        self,
+        name: str,
+        annotation: Optional[str],
+        default: Optional[str],
+        kind: ParameterKind,
+    ) -> None:
+        self.name = name
+        self.kind = kind
+
+        self.annotation = annotation
+        self.default = default
+
+    def __repr__(self) -> str:
+        return f"Parameter(name={self.name!r})"
+
+    @property
+    def description(self) -> str:
+        """
+        Name + annotation.
+        """
+        description = self.name
+
+        if self.annotation is not None:
+            description += f": {self.annotation}"
+
+        return description
+
+
+class Signature:
+    """
+    Signature definition used wrap around both Jedi signatures and
+    python-inspect signatures.
+
+    :param index: Parameter index of the current cursor position.
+    :param bracket_start: (line, column) tuple for the open bracket that starts
+        the function call.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        docstring: str,
+        parameters: Sequence[Parameter],
+        index: Optional[int] = None,
+        returns: str = "",
+        bracket_start: Tuple[int, int] = (0, 0),
+    ) -> None:
+        self.name = name
+        self.docstring = docstring
+        self.parameters = parameters
+        self.index = index
+        self.returns = returns
+        self.bracket_start = bracket_start
+
+    @classmethod
+    def from_inspect_signature(
+        cls,
+        name: str,
+        docstring: str,
+        signature: InspectSignature,
+        index: int,
+    ) -> "Signature":
+        parameters = []
+
+        def get_annotation_name(annotation: object) -> str:
+            """
+            Get annotation as string from inspect signature.
+            """
+            try:
+                # In case the annotation is a class like "int", "float", ...
+                return str(annotation.__name__)  # type: ignore
+            except AttributeError:
+                pass  # No attribute `__name__`, e.g., in case of `List[int]`.
+
+            annotation = str(annotation)
+            if annotation.startswith("typing."):
+                annotation = annotation[len("typing:") :]
+            return annotation
+
+        for p in signature.parameters.values():
+            parameters.append(
+                Parameter(
+                    name=p.name,
+                    annotation=get_annotation_name(p.annotation),
+                    default=repr(p.default)
+                    if p.default is not inspect.Parameter.empty
+                    else None,
+                    kind=p.kind,
+                )
+            )
+
+        return cls(
+            name=name,
+            docstring=docstring,
+            parameters=parameters,
+            index=index,
+            returns="",
+        )
+
+    @classmethod
+    def from_jedi_signature(cls, signature) -> "Signature":
+        parameters = []
+
+        for p in signature.params:
+            if p is None:
+                # We just hit the "*".
+                continue
+
+            parameters.append(
+                Parameter(
+                    name=p.to_string(),  # p.name, (`to_string()` already includes the annotation).
+                    annotation=None,  # p.infer_annotation()
+                    default=None,  # p.infer_default()
+                    kind=p.kind,
+                )
+            )
+
+        docstring = signature.docstring()
+        if not isinstance(docstring, str):
+            docstring = docstring.decode("utf-8")
+
+        return cls(
+            name=signature.name,
+            docstring=docstring,
+            parameters=parameters,
+            index=signature.index,
+            returns="",
+            bracket_start=signature.bracket_start,
+        )
+
+    def __repr__(self) -> str:
+        return f"Signature({self.name!r}, parameters={self.parameters!r})"
+
+
+def get_signatures_using_jedi(
+    document: Document, locals: Dict[str, Any], globals: Dict[str, Any]
+) -> List[Signature]:
+    script = get_jedi_script_from_document(document, locals, globals)
+
+    # Show signatures in help text.
+    if not script:
+        return []
+
+    try:
+        signatures = script.get_signatures()
+    except ValueError:
+        # e.g. in case of an invalid \\x escape.
+        signatures = []
+    except Exception:
+        # Sometimes we still get an exception (TypeError), because
+        # of probably bugs in jedi. We can silence them.
+        # See: https://github.com/davidhalter/jedi/issues/492
+        signatures = []
+    else:
+        # Try to access the params attribute just once. For Jedi
+        # signatures containing the keyword-only argument star,
+        # this will crash when retrieving it the first time with
+        # AttributeError. Every following time it works.
+        # See: https://github.com/jonathanslenders/ptpython/issues/47
+        #      https://github.com/davidhalter/jedi/issues/598
+        try:
+            if signatures:
+                signatures[0].params
+        except AttributeError:
+            pass
+
+    return [Signature.from_jedi_signature(sig) for sig in signatures]
+
+
+def get_signatures_using_eval(
+    document: Document, locals: Dict[str, Any], globals: Dict[str, Any]
+) -> List[Signature]:
+    """
+    Look for the signature of the function before the cursor position without
+    use of Jedi. This uses a similar approach as the `DictionaryCompleter` of
+    running `eval()` over the detected function name.
+    """
+    # Look for open parenthesis, before cursor position.
+    text = document.text_before_cursor
+    pos = document.cursor_position - 1
+
+    paren_mapping = {")": "(", "}": "{", "]": "["}
+    paren_stack = [
+        ")"
+    ]  # Start stack with closing ')'. We are going to look for the matching open ')'.
+    comma_count = 0  # Number of comma's between start of function call and cursor pos.
+    found_start = False  # Found something.
+
+    while pos >= 0:
+        char = document.text[pos]
+        if char in ")]}":
+            paren_stack.append(char)
+        elif char in "([{":
+            if not paren_stack:
+                # Open paren, while no closing paren was found. Mouse cursor is
+                # positioned in nested parentheses. Not at the "top-level" of a
+                # function call.
+                break
+            if paren_mapping[paren_stack[-1]] != char:
+                # Unmatching parentheses: syntax error?
+                break
+
+            paren_stack.pop()
+
+            if len(paren_stack) == 0:
+                found_start = True
+                break
+
+        elif char == "," and len(paren_stack) == 1:
+            comma_count += 1
+
+        pos -= 1
+
+    if not found_start:
+        return []
+
+    # We found the start of the function call. Now look for the object before
+    # this position on which we can do an 'eval' to retrieve the function
+    # object.
+    obj = DictionaryCompleter(lambda: globals, lambda: locals).eval_expression(
+        Document(document.text, cursor_position=pos), locals
+    )
+    if obj is None:
+        return []
+
+    try:
+        name = obj.__name__  # type:ignore
+    except Exception:
+        name = obj.__class__.__name__
+
+    try:
+        signature = inspect.signature(obj)  # type: ignore
+    except TypeError:
+        return []  # Not a callable object.
+    except ValueError:
+        return []  # No signature found, like for build-ins like "print".
+
+    try:
+        doc = obj.__doc__ or ""
+    except:
+        doc = ""
+
+    # TODO: `index` is not yet correct when dealing with keyword-only arguments.
+    return [Signature.from_inspect_signature(name, doc, signature, index=comma_count)]

ptpython/style.py

@@ -0,0 +1,175 @@
+from typing import Dict
+
+from prompt_toolkit.styles import BaseStyle, Style, merge_styles
+from prompt_toolkit.styles.pygments import style_from_pygments_cls
+from prompt_toolkit.utils import is_conemu_ansi, is_windows, is_windows_vt100_supported
+from pygments.styles import get_all_styles, get_style_by_name
+
+__all__ = ["get_all_code_styles", "get_all_ui_styles", "generate_style"]
+
+
+def get_all_code_styles() -> Dict[str, BaseStyle]:
+    """
+    Return a mapping from style names to their classes.
+    """
+    result: Dict[str, BaseStyle] = {
+        name: style_from_pygments_cls(get_style_by_name(name))
+        for name in get_all_styles()
+    }
+    result["win32"] = Style.from_dict(win32_code_style)
+    return result
+
+
+def get_all_ui_styles() -> Dict[str, BaseStyle]:
+    """
+    Return a dict mapping {ui_style_name -> style_dict}.
+    """
+    return {
+        "default": Style.from_dict(default_ui_style),
+        "blue": Style.from_dict(blue_ui_style),
+    }
+
+
+def generate_style(python_style: BaseStyle, ui_style: BaseStyle) -> BaseStyle:
+    """
+    Generate Pygments Style class from two dictionaries
+    containing style rules.
+    """
+    return merge_styles([python_style, ui_style])
+
+
+# Code style for Windows consoles. They support only 16 colors,
+# so we choose a combination that displays nicely.
+win32_code_style = {
+    "pygments.comment": "#00ff00",
+    "pygments.keyword": "#44ff44",
+    "pygments.number": "",
+    "pygments.operator": "",
+    "pygments.string": "#ff44ff",
+    "pygments.name": "",
+    "pygments.name.decorator": "#ff4444",
+    "pygments.name.class": "#ff4444",
+    "pygments.name.function": "#ff4444",
+    "pygments.name.builtin": "#ff4444",
+    "pygments.name.attribute": "",
+    "pygments.name.constant": "",
+    "pygments.name.entity": "",
+    "pygments.name.exception": "",
+    "pygments.name.label": "",
+    "pygments.name.namespace": "",
+    "pygments.name.tag": "",
+    "pygments.name.variable": "",
+}
+
+
+default_ui_style = {
+    "control-character": "ansiblue",
+    # Classic prompt.
+    "prompt": "bold",
+    "prompt.dots": "noinherit",
+    # (IPython <5.0) Prompt: "In [1]:"
+    "in": "bold #008800",
+    "in.number": "",
+    # Return value.
+    "out": "#ff0000",
+    "out.number": "#ff0000",
+    # Completions.
+    "completion.builtin": "",
+    "completion.param": "#006666 italic",
+    "completion.keyword": "fg:#008800",
+    "completion.keyword fuzzymatch.inside": "fg:#008800",
+    "completion.keyword fuzzymatch.outside": "fg:#44aa44",
+    # Separator between windows. (Used above docstring.)
+    "separator": "#bbbbbb",
+    # System toolbar
+    "system-toolbar": "#22aaaa noinherit",
+    # "arg" toolbar.
+    "arg-toolbar": "#22aaaa noinherit",
+    "arg-toolbar.text": "noinherit",
+    # Signature toolbar.
+    "signature-toolbar": "bg:#44bbbb #000000",
+    "signature-toolbar current-name": "bg:#008888 #ffffff bold",
+    "signature-toolbar operator": "#000000 bold",
+    "docstring": "#888888",
+    # Validation toolbar.
+    "validation-toolbar": "bg:#440000 #aaaaaa",
+    # Status toolbar.
+    "status-toolbar": "bg:#222222 #aaaaaa",
+    "status-toolbar.title": "underline",
+    "status-toolbar.inputmode": "bg:#222222 #ffffaa",
+    "status-toolbar.key": "bg:#000000 #888888",
+    "status-toolbar key": "bg:#000000 #888888",
+    "status-toolbar.pastemodeon": "bg:#aa4444 #ffffff",
+    "status-toolbar.pythonversion": "bg:#222222 #ffffff bold",
+    "status-toolbar paste-mode-on": "bg:#aa4444 #ffffff",
+    "record": "bg:#884444 white",
+    "status-toolbar more": "#ffff44",
+    "status-toolbar.input-mode": "#ffff44",
+    # The options sidebar.
+    "sidebar": "bg:#bbbbbb #000000",
+    "sidebar.title": "bg:#668866 #ffffff",
+    "sidebar.label": "bg:#bbbbbb #222222",
+    "sidebar.status": "bg:#dddddd #000011",
+    "sidebar.label selected": "bg:#222222 #eeeeee",
+    "sidebar.status selected": "bg:#444444 #ffffff bold",
+    "sidebar.separator": "underline",
+    "sidebar.key": "bg:#bbddbb #000000 bold",
+    "sidebar.key.description": "bg:#bbbbbb #000000",
+    "sidebar.helptext": "bg:#fdf6e3 #000011",
+    #        # Styling for the history layout.
+    #        history.line:                          '',
+    #        history.line.selected:                 'bg:#008800  #000000',
+    #        history.line.current:                  'bg:#ffffff #000000',
+    #        history.line.selected.current:         'bg:#88ff88 #000000',
+    #        history.existinginput:                  '#888888',
+    # Help Window.
+    "window-border": "#aaaaaa",
+    "window-title": "bg:#bbbbbb #000000",
+    # Meta-enter message.
+    "accept-message": "bg:#ffff88 #444444",
+    # Exit confirmation.
+    "exit-confirmation": "bg:#884444 #ffffff",
+}
+
+
+# Some changes to get a bit more contrast on Windows consoles.
+# (They only support 16 colors.)
+if is_windows() and not is_conemu_ansi() and not is_windows_vt100_supported():
+    default_ui_style.update(
+        {
+            "sidebar.title": "bg:#00ff00 #ffffff",
+            "exitconfirmation": "bg:#ff4444 #ffffff",
+            "toolbar.validation": "bg:#ff4444 #ffffff",
+            "menu.completions.completion": "bg:#ffffff #000000",
+            "menu.completions.completion.current": "bg:#aaaaaa #000000",
+        }
+    )
+
+
+blue_ui_style = {}
+blue_ui_style.update(default_ui_style)
+# blue_ui_style.update({
+#        # Line numbers.
+#        Token.LineNumber:                             '#aa6666',
+#
+#        # Highlighting of search matches in document.
+#        Token.SearchMatch:                            '#ffffff bg:#4444aa',
+#        Token.SearchMatch.Current:                    '#ffffff bg:#44aa44',
+#
+#        # Highlighting of select text in document.
+#        Token.SelectedText:                           '#ffffff bg:#6666aa',
+#
+#        # Completer toolbar.
+#        Token.Toolbar.Completions:                    'bg:#44bbbb #000000',
+#        Token.Toolbar.Completions.Arrow:              'bg:#44bbbb #000000 bold',
+#        Token.Toolbar.Completions.Completion:         'bg:#44bbbb #000000',
+#        Token.Toolbar.Completions.Completion.Current: 'bg:#008888 #ffffff',
+#
+#        # Completer menu.
+#        Token.Menu.Completions.Completion:            'bg:#44bbbb #000000',
+#        Token.Menu.Completions.Completion.Current:    'bg:#008888 #ffffff',
+#        Token.Menu.Completions.Meta:                  'bg:#449999 #000000',
+#        Token.Menu.Completions.Meta.Current:          'bg:#00aaaa #000000',
+#        Token.Menu.Completions.ProgressBar:           'bg:#aaaaaa',
+#        Token.Menu.Completions.ProgressButton:        'bg:#000000',
+# })

ptpython/utils.py

@@ -0,0 +1,198 @@
+"""
+For internal use only.
+"""
+import re
+from typing import Callable, Iterable, Type, TypeVar, cast
+
+from prompt_toolkit.formatted_text import to_formatted_text
+from prompt_toolkit.formatted_text.utils import fragment_list_to_text
+from prompt_toolkit.mouse_events import MouseEvent, MouseEventType
+
+__all__ = [
+    "has_unclosed_brackets",
+    "get_jedi_script_from_document",
+    "document_is_multiline_python",
+    "unindent_code",
+]
+
+
+def has_unclosed_brackets(text: str) -> bool:
+    """
+    Starting at the end of the string. If we find an opening bracket
+    for which we didn't had a closing one yet, return True.
+    """
+    stack = []
+
+    # Ignore braces inside strings
+    text = re.sub(r"""('[^']*'|"[^"]*")""", "", text)  # XXX: handle escaped quotes.!
+
+    for c in reversed(text):
+        if c in "])}":
+            stack.append(c)
+
+        elif c in "[({":
+            if stack:
+                if (
+                    (c == "[" and stack[-1] == "]")
+                    or (c == "{" and stack[-1] == "}")
+                    or (c == "(" and stack[-1] == ")")
+                ):
+                    stack.pop()
+            else:
+                # Opening bracket for which we didn't had a closing one.
+                return True
+
+    return False
+
+
+def get_jedi_script_from_document(document, locals, globals):
+    import jedi  # We keep this import in-line, to improve start-up time.
+
+    # Importing Jedi is 'slow'.
+
+    try:
+        return jedi.Interpreter(
+            document.text,
+            path="input-text",
+            namespaces=[locals, globals],
+        )
+    except ValueError:
+        # Invalid cursor position.
+        # ValueError('`column` parameter is not in a valid range.')
+        return None
+    except AttributeError:
+        # Workaround for #65: https://github.com/jonathanslenders/python-prompt-toolkit/issues/65
+        # See also: https://github.com/davidhalter/jedi/issues/508
+        return None
+    except IndexError:
+        # Workaround Jedi issue #514: for https://github.com/davidhalter/jedi/issues/514
+        return None
+    except KeyError:
+        # Workaroud for a crash when the input is "u'", the start of a unicode string.
+        return None
+    except Exception:
+        # Workaround for: https://github.com/jonathanslenders/ptpython/issues/91
+        return None
+
+
+_multiline_string_delims = re.compile("""[']{3}|["]{3}""")
+
+
+def document_is_multiline_python(document):
+    """
+    Determine whether this is a multiline Python document.
+    """
+
+    def ends_in_multiline_string() -> bool:
+        """
+        ``True`` if we're inside a multiline string at the end of the text.
+        """
+        delims = _multiline_string_delims.findall(document.text)
+        opening = None
+        for delim in delims:
+            if opening is None:
+                opening = delim
+            elif delim == opening:
+                opening = None
+        return bool(opening)
+
+    if "\n" in document.text or ends_in_multiline_string():
+        return True
+
+    def line_ends_with_colon() -> bool:
+        return document.current_line.rstrip()[-1:] == ":"
+
+    # If we just typed a colon, or still have open brackets, always insert a real newline.
+    if (
+        line_ends_with_colon()
+        or (
+            document.is_cursor_at_the_end
+            and has_unclosed_brackets(document.text_before_cursor)
+        )
+        or document.text.startswith("@")
+    ):
+        return True
+
+    # If the character before the cursor is a backslash (line continuation
+    # char), insert a new line.
+    elif document.text_before_cursor[-1:] == "\\":
+        return True
+
+    return False
+
+
+_T = TypeVar("_T", bound=Callable[[MouseEvent], None])
+
+
+def if_mousedown(handler: _T) -> _T:
+    """
+    Decorator for mouse handlers.
+    Only handle event when the user pressed mouse down.
+
+    (When applied to a token list. Scroll events will bubble up and are handled
+    by the Window.)
+    """
+
+    def handle_if_mouse_down(mouse_event: MouseEvent):
+        if mouse_event.event_type == MouseEventType.MOUSE_DOWN:
+            return handler(mouse_event)
+        else:
+            return NotImplemented
+
+    return cast(_T, handle_if_mouse_down)
+
+
+_T_type = TypeVar("_T_type", bound=Type)
+
+
+def ptrepr_to_repr(cls: _T_type) -> _T_type:
+    """
+    Generate a normal `__repr__` method for classes that have a `__pt_repr__`.
+    """
+    if not hasattr(cls, "__pt_repr__"):
+        raise TypeError(
+            "@ptrepr_to_repr can only be applied to classes that have a `__pt_repr__` method."
+        )
+
+    def __repr__(self) -> str:
+        return fragment_list_to_text(to_formatted_text(cls.__pt_repr__(self)))
+
+    cls.__repr__ = __repr__  # type:ignore
+    return cls
+
+
+def unindent_code(text: str) -> str:
+    """
+    Remove common leading whitespace when all lines are indented.
+    """
+    lines = text.splitlines(keepends=True)
+
+    # Look for common prefix.
+    common_prefix = _common_whitespace_prefix(lines)
+
+    # Remove indentation.
+    lines = [line[len(common_prefix) :] for line in lines]
+
+    return "".join(lines)
+
+
+def _common_whitespace_prefix(strings: Iterable[str]) -> str:
+    """
+    Return common prefix for a list of lines.
+    This will ignore lines that contain whitespace only.
+    """
+    # Ignore empty lines and lines that have whitespace only.
+    strings = [s for s in strings if not s.isspace() and not len(s) == 0]
+
+    if not strings:
+        return ""
+
+    else:
+        s1 = min(strings)
+        s2 = max(strings)
+
+        for i, c in enumerate(s1):
+            if c != s2[i] or c not in " \t":
+                return s1[:i]
+
+        return s1

ptpython/validator.py

@@ -0,0 +1,57 @@
+from prompt_toolkit.validation import ValidationError, Validator
+
+from .utils import unindent_code
+
+__all__ = ["PythonValidator"]
+
+
+class PythonValidator(Validator):
+    """
+    Validation of Python input.
+
+    :param get_compiler_flags: Callable that returns the currently
+        active compiler flags.
+    """
+
+    def __init__(self, get_compiler_flags=None):
+        self.get_compiler_flags = get_compiler_flags
+
+    def validate(self, document):
+        """
+        Check input for Python syntax errors.
+        """
+        text = unindent_code(document.text)
+
+        # When the input starts with Ctrl-Z, always accept. This means EOF in a
+        # Python REPL.
+        if text.startswith("\x1a"):
+            return
+
+        # When the input starts with an exclamation mark. Accept as shell
+        # command.
+        if text.lstrip().startswith("!"):
+            return
+
+        try:
+            if self.get_compiler_flags:
+                flags = self.get_compiler_flags()
+            else:
+                flags = 0
+
+            compile(text, "<input>", "exec", flags=flags, dont_inherit=True)
+        except SyntaxError as e:
+            # Note, the 'or 1' for offset is required because Python 2.7
+            # gives `None` as offset in case of '4=4' as input. (Looks like
+            # fixed in Python 3.)
+            # TODO: This is not correct if indentation was removed.
+            index = document.translate_row_col_to_index(
+                e.lineno - 1, (e.offset or 1) - 1
+            )
+            raise ValidationError(index, f"Syntax Error: {e}")
+        except TypeError as e:
+            # e.g. "compile() expected string without null bytes"
+            raise ValidationError(0, str(e))
+        except ValueError as e:
+            # In Python 2, compiling "\x9" (an invalid escape sequence) raises
+            # ValueError instead of SyntaxError.
+            raise ValidationError(0, "Syntax Error: %s" % e)

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.black]
+target-version = ['py36']
+
+
+[tool.isort]
+# isort configuration that is compatible with Black.
+multi_line_output = 3
+include_trailing_comma = true
+known_first_party = "ptpython"
+known_third_party = "prompt_toolkit,pygments,asyncssh"
+force_grid_wrap = 0
+use_parentheses = true
+line_length = 88

setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1

setup.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+import os
+import sys
+
+from setuptools import find_packages, setup
+
+with open(os.path.join(os.path.dirname(__file__), "README.rst")) as f:
+    long_description = f.read()
+
+
+setup(
+    name="ptpython",
+    author="Jonathan Slenders",
+    version="3.0.16",
+    url="https://github.com/prompt-toolkit/ptpython",
+    description="Python REPL build on top of prompt_toolkit",
+    long_description=long_description,
+    packages=find_packages("."),
+    install_requires=[
+        "appdirs",
+        "importlib_metadata;python_version<'3.8'",
+        "jedi>=0.16.0",
+        # Use prompt_toolkit 3.0.16, because of the `DeduplicateCompleter`.
+        "prompt_toolkit>=3.0.16,<3.1.0",
+        "pygments",
+        "black",
+    ],
+    python_requires=">=3.6",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python",
+    ],
+    entry_points={
+        "console_scripts": [
+            "ptpython = ptpython.entry_points.run_ptpython:run",
+            "ptipython = ptpython.entry_points.run_ptipython:run",
+            "ptpython%s = ptpython.entry_points.run_ptpython:run" % sys.version_info[0],
+            "ptpython%s.%s = ptpython.entry_points.run_ptpython:run"
+            % sys.version_info[:2],
+            "ptipython%s = ptpython.entry_points.run_ptipython:run"
+            % sys.version_info[0],
+            "ptipython%s.%s = ptpython.entry_points.run_ptipython:run"
+            % sys.version_info[:2],
+        ]
+    },
+    extras_require={"ptipython": ["ipython"]},  # For ptipython, we need to have IPython
+)

tests/run_tests.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import unittest
+
+import ptpython.completer
+import ptpython.eventloop
+import ptpython.filters
+import ptpython.history_browser
+import ptpython.key_bindings
+import ptpython.layout
+import ptpython.python_input
+import ptpython.repl
+import ptpython.style
+import ptpython.utils
+import ptpython.validator
+
+# For now there are no tests here.
+# However this is sufficient for Travis to do at least a syntax check.
+# That way we are at least sure to restrict to the Python 2.6 syntax.
+
+
+if __name__ == "__main__":
+    unittest.main()