# [scn_base.h] Basic Token Separation

## contents

#include "symbols.h"
#include "scn_io.h"

The module [scn_base] implements the scan stream interface based on a given STYX-konform scanner definition. ( see [scn_gen] )

The scan stream is reentrant and allows multiple scan operations at a time.

## Macros

Token definition flags ( see Scn_dfnToken )

    #define SCN_FLG_IgnoreToken 1  /* token has to be ignored          */
    #define SCN_FLG_IgnoreCase  2  /* case insensitive token           */
    #define SCN_FLG_EofToken    4  /* eof token in embedded language   */
    #define SCN_FLG_SwitchToken 8  /* switch to/from embedded language */
    #define SCN_FLG_CfgToken    16 /* embedded language token          */
    #define SCN_FLG_IndentToken 32 /* (de)indent token                 */

Token preprocessing result values

    #define SCN_FLG_RescanNone     0
    #define SCN_FLG_RescanString   1
    #define SCN_FLG_RescanBString  2
    #define SCN_FLG_RescanFile     3
    #define SCN_FLG_RescanBFile    4
    #define SCN_FLG_RescanExtFile  5
    #define SCN_FLG_RescanExtBFile 6
    #define SCN_FLG_RescanChar     7
    #define SCN_FLG_RescanChrItr   8
    #define SCN_FLG_RescanBChrItr  9

## Stream of tokens

Scn_Stream | Stream onto a scanner

Scn_Stream_Itr | Character iterator onto a scan stream

       Type of token preprocessing function
       RC = SCN_FLG_RescanChrItr   & valid cMacVal --> rescan character iterator cMacVal
                                                       ( frees character iterator )
       RC = SCN_FLG_RescanBChrItr  & valid cMacVal --> rescan binary char iterator cMacVal
                                                       ( frees character iterator )
       RC = SCN_FLG_RescanString   & valid cMacVal --> rescan string cMacVal
       RC = SCN_FLG_RescanBString  & valid cMacVal --> rescan binary string cMacVal
       RC = SCN_FLG_RescanBFile    & valid cMacVal --> rescan binary file cMacVal
       RC = SCN_FLG_RescanExtFile  & valid cMacVal --> rescan file cMacVal,
                                                       with main stream read function
       RC = SCN_FLG_RescanExtBFile & valid cMacVal --> rescan binary file cMacVal,
                                                       with main stream read function
       RC = SCN_FLG_RescanChar     & valid cMacVal --> rescan character cMacVal
                                                       ( single byte or ucs4 )
       RC = SCN_FLG_RescanNone     & valid cMacVal --> replace cTokVal with cMacVal
                                                       ( single byte or utf-8 )
       ELSE                                        --> ignore cTokVal

    typedef int (*Scn_PreMacFun)
                (
                  Scn_Stream pStream, c_string cTokNam,
                  c_string cTokVal, symbol* cMacVal
                );

Type of the embedded language token constructor

    typedef Any_T (*Scn_eTerm)
                  (
                    Abs_T CurPTCfg, c_string language, c_string StartSymbol
                  );

Type of the embedded language accept token recognizer

    typedef c_bool (*Scn_eAccept)(Abs_T CurPTCfg);

## Accessing scanner definition

    c_string Scn_id(Scn_T scn)
| name of the scanner (group) 'scn'; allocs memory
    short Scn_check_Token(Scn_T scn, c_string s)
| whether string 's' is a token in scanner 'scn'; result = token number ( > 0 ) or 0 assertion: Scn_groups(scn) = 0
    short Scn_check_WCToken(Scn_T scn, wc_string s)
| whether wide string 's' is a token in scanner 'scn'; result = token number ( > 0 ) or 0 assertion: Scn_groups(scn) = 0
    int Scn_tokens(Scn_T scn)
| number of tokens defined in scanner ( group ) 'scn'
    c_string Scn_tokid(Scn_T scn, int i)
| name of the i-th token definied in scanner 'scn'; allocs memory assertion: Scn_groups(scn) = 0
    c_byte Scn_tokFlags(Scn_T scn, int i)
| flags of the i-th token definied in scanner 'scn' assertion: Scn_groups(scn) = 0
    int Scn_dycks(Scn_T scn)
| number of dyck token defined in scanner ( group ) 'scn'
    Scn_T Scn_dyck(Scn_T scn, int i)
| i-th dyck scanner definied in scanner group 'scn' assertion: Scn_dycks(scn) > 0
    int Scn_dycktoken(Scn_T scn, int i)
| i-th dyck token definied in scanner group 'scn' assertion: Scn_dycks(scn) > 0
    int Scn_groups(Scn_T scn)
| number of groups defined in scanner ( group ) 'scn'
    Scn_T Scn_group(Scn_T scn, int i)
| i-th scanner definied in scanner group 'scn' assertion: Scn_groups(scn) > 0
    int Scn_SwitchGroup(Scn_T scn, int i)
| switch group of the i-th token definied in scanner group 'scn' or -1 assertion: Scn_groups(scn) = 0
    long Scn_check_GroupToken(Scn_T scn, c_string s)
| whether string 's' is a token in scanner group 'scn'; result: high = group index , low = token number ( > 0 ) or 0 assertion: Scn_groups(scn) > 0
    long Scn_check_GroupWCToken(Scn_T scn, wc_string s)
| whether wide string 's' is a token in scanner group 'scn'; result: high = group index , low = token number ( > 0 ) or 0 assertion: Scn_groups(scn) > 0

## Character iterator definition

    Scn_Stream_Itr Stream_Itr_new
                   (
                     int      f_getc (StdCPtr file),
                     void     f_close(StdCPtr file),
                     c_string   f_wc2mb(wc_string wc),
                     LONG_INT f_seek (StdCPtr file, long offset, int origin),
                     StdCPtr  file,
                     c_string   fileid
                   )
| creates and initializes a character iterator onto the open character source 'file' 'f_getc' : next character 'f_close': closes character source 'f_wc2mb': converts wide to multibyte character 'f_seek' : positions character source 'fileid' : character source identifier
    void Stream_Itr_free(Scn_Stream_Itr itr)
| frees character iterator 'itr'

## Scan stream definition

    Scn_Stream Stream_bgn(Scn_T scn, Scn_Stream_Itr itr)
| creates and initializes a scan stream onto scanner 'scn' and character iterator 'itr'
    void Stream_close(Scn_Stream t)
| closes character source i.e. file of scan stream 't'
    void Stream_free(Scn_Stream t)
| frees scan stream 't'
    void Stream_reset(Scn_Stream t)
| re-initializes scan process in current scan stream 't'
    void Stream_premac_set(Scn_Stream t, Scn_PreMacFun cPreMac)
| adds 'cPreMac' to scan stream 't' for token preprocessing & macro expansion
    void Stream_eterm_set
         (
           Scn_Stream t, Abs_T CurPTCfg, Scn_eTerm eTerm, Scn_eAccept eAccept
         )
| adds 'eTerm', 'eAccept' and 'CurPTCfg' to scan stream 't' for embedded language token construction ( deactivates other token concatenation )
    void Stream_add_ctxval(Scn_Stream t, Any_T id, Any_T val)
| adds the context pair 'id' |--> 'val' to scan stream 't'; updates existing entries
    void Stream_concat_other(Scn_Stream t)
| activates other token concatenation in scan stream 't'
    void Stream_skip_character(Scn_Stream t, int cC)
| skips character 'cC' ( but tracks position )
    void Stream_binmode_set(Scn_Stream t)
| activates binary mode in scan stream 't' ( deactivates other token concatenation )
    Scn_Stream Stream_line
               (
                 Scn_T scn, StdCPtr line, int (*cGet)(StdCPtr line), c_string id
               )
    #define STREAM_STDIN(scn) \
    	Stream_line(scn,stdin,(int (*)(StdCPtr f))fgetc,"<stdin>")
| creates a scan stream onto scanner 'scn' and character stream 'line' 'cGet': next character 'id' : character stream identifier
    Scn_Stream Stream_file(Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext)
| creates a scan stream onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext'
    Scn_Stream Stream_extfile
               (
                 Scn_T scn, c_string EnvVar, c_string FileName, c_string Ext,
                 int (*cGet)(FILE* f)
               )
| creates a scan stream onto scanner 'scn' and file [$'EnvVar'/]'FileName''Ext' with external next character function 'cGet'
    Scn_Stream Stream_bstring(Scn_T scn, c_bstring bstr)
| creates a scan stream onto scanner 'scn' and binary string 'bstr'
    void Stream_bstring_set(Scn_Stream stream, c_bstring bstr)
| re-initializes scan stream 'stream' with binary string 'bstr'
    Scn_Stream Stream_string(Scn_T scn, c_string Text)
| creates a scan stream onto scanner 'scn' and string 'Text'
    void Stream_string_set(Scn_Stream stream, c_string Text)
| re-initializes scan stream 'stream' with string 'Text'
    void Stream_string_insert(Scn_Stream stream, c_string Text)
| inserts string 'Text' at current position in scan stream 'stream'

## Defining token numbers

    void Stream_defEofId(Scn_Stream t, short tid)
| defines 'tid' as EOF token number in scan stream 't'
    void Stream_defErrId(Scn_Stream t, short tid)
| defines 'tid' as error token number in scan stream 't'
    void Stream_defTokId(Scn_Stream t, c_string Name, short tid)
| defines 'tid' as number for token 'Name' in scan stream 't'
    void Stream_defKeyId(Scn_Stream t, c_string Text, short tid)
| defines 'tid' as number for keyword 'Text' in scan stream 't'
    void Stream_defWCKeyId(Scn_Stream t, wc_string Text, short tid)
| defines 'tid' as number for wide keyword 'Text' in scan stream 't'

## Scan stream operations

    void Stream_next(Scn_Stream t)
| separates the next token in scan stream 't'; That's the workhorse operation.

## Accessing current token

    symbol Stream_csym(Scn_Stream t)
| symbol of the current token in scan stream 't'
    short Stream_ctid(Scn_Stream t)
| token number of the current token in scan stream 't'
    c_string Stream_ctnam(Scn_Stream t)
| name of the current token in scan stream 't' or NULL (EOF)
    c_bool Stream_cica(Scn_Stream t)
| whether the current token in scan stream 't' has the attribute 'ignore case'
    c_string Stream_cbuf(Scn_Stream t)
| string of the current token in scan stream 't' or NULL ( not valid for an embedded language token )
    wc_string Stream_wcbuf(Scn_Stream t)
| wide string of the current token in scan stream 't' or NULL ( not valid for an embedded language token )
    long Stream_clen(Scn_Stream t)
| length of the current token in scan stream 't' ( not valid for an embedded language and the follow token )
    symbol Stream_cfil(Scn_Stream t)
| source identifier of the current token in scan stream 't'
    long  Stream_clin(Scn_Stream t)
| first line number of the current token in scan stream 't'
    long  Stream_ccol(Scn_Stream t)
| first column number of the current token in scan stream 't'

## Accessing scan stream definition

    Scn_T Stream_scn(Scn_Stream t)
| scanner definition of scan stream 't'
    Scn_PreMacFun Stream_premac(Scn_Stream t)
| preprocessor of scan stream 't'
    Scn_eTerm Stream_etermFun(Scn_Stream t)
| embedded language token constructor of scan stream 't'
    Scn_eAccept Stream_etermEof(Scn_Stream t)
| embedded language accept token recognizer of scan stream 't'
    Abs_T Stream_etermCfg(Scn_Stream t)
| current parse term configuration of scan stream 't'
    c_bool Stream_binmode(Scn_Stream t)
| whether current scan stream 't' has binary mode
    c_bool Stream_unicode(Scn_Stream t)
| whether current scan stream 't' is unicode-based
    StdCPtr Stream_cstream(Scn_Stream t)
| character stream of current scan stream 't'
    Any_T Stream_get_ctxval(Scn_Stream t, Any_T id)
| value of context variable 'id' in scan stream 't' or NULL