lex.fsl

// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.

{

module internal Microsoft.FSharp.Compiler.Lexer

//------------------------------------------------------------------------
// The Lexer.  Some of the complication arises from the fact it is 
// reused by the Visual Studio mode to do partial lexing reporting 
// whitespace etc.
//-----------------------------------------------------------------------

open System
open System.Globalization
open System.Text
open Internal.Utilities

open Microsoft.FSharp.Compiler.AbstractIL
open Microsoft.FSharp.Compiler.AbstractIL.Internal
open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library
open Microsoft.FSharp.Compiler

open Microsoft.FSharp.Compiler.Range
open Microsoft.FSharp.Compiler.Ast
open Microsoft.FSharp.Compiler.ErrorLogger
open Microsoft.FSharp.Compiler.Parser
open Microsoft.FSharp.Compiler.Lexhelp
open Microsoft.FSharp.Compiler.Lib
open Internal.Utilities.Text.Lexing

let lexeme (lexbuf : UnicodeLexing.Lexbuf) = UnicodeLexing.Lexbuf.LexemeString lexbuf

let trimBoth (s:string) n m = s.Substring(n, s.Length - (n+m))
let lexemeTrimBoth   lexbuf n m = trimBoth (lexeme lexbuf) n m
let lexemeTrimRight  lexbuf n = lexemeTrimBoth lexbuf 0 n
let lexemeTrimLeft   lexbuf n = lexemeTrimBoth lexbuf n 0

let fail args (lexbuf:UnicodeLexing.Lexbuf) msg dflt =
     let m = lexbuf.LexemeRange
     args.errorLogger.ErrorR(Error(msg,m))
     dflt        

//--------------------------
// Integer parsing

// Parsing integers is common in bootstrap runs (parsing
// the parser tables, no doubt). So this is an optimized
// version of the F# core library parsing code with the call to "Trim"
// removed, which appears in profiling runs as a small but significant cost.

let getSign32 (s:string) (p:byref<int>) l = 
    if (l >= p + 1 && s.[p] = '-') 
    then p <- p + 1; -1 
    else 1 

let isOXB c = 
    let c = Char.ToLowerInvariant c
    c = 'x' || c = 'o' || c = 'b'

let is0OXB (s:string) p l = 
    l >= p + 2 && s.[p] = '0' && isOXB s.[p+1]
let get0OXB (s:string) (p:byref<int>)  l = 
    if is0OXB s p l
    then let r = Char.ToLowerInvariant s.[p+1] in p <- p + 2; r
    else 'd' 

let formatError() = raise (new System.FormatException(SR.GetString("bad format string")))

let parseBinaryUInt64 (s:string) p l = 
    let rec parse n acc = if n < l then parse (n+1) (acc * 2UL + (match s.[n] with '0' -> 0UL | '1' -> 1UL | _ -> formatError())) else acc 
    parse p 0UL

let parseOctalUInt64 (s:string) p l = 
    let rec parse n acc = if n < l then parse (n+1) (acc * 8UL + (let c = s.[n] in if c >= '0' && c <= '7' then Convert.ToUInt64 c - Convert.ToUInt64 '0' else formatError())) else acc 
    parse p 0UL

let removeUnderscores (s:string) =
    match s with
    | null -> null
    | s -> s.Replace("_", "")

let parseInt32 (s:string) = 
    let s = removeUnderscores s
    let l = s.Length 
    let mutable p = 0 
    let sign = getSign32 s &p l 
    let specifier = get0OXB s &p l 
#if FX_RESHAPED_GLOBALIZATION
    match CultureInfo.InvariantCulture.TextInfo.ToLower(specifier) with 
#else
    match Char.ToLower(specifier,CultureInfo.InvariantCulture) with 
#endif
    | 'x' -> sign * (int32 (Convert.ToUInt32(UInt64.Parse(s.Substring(p), NumberStyles.AllowHexSpecifier,CultureInfo.InvariantCulture))))
    | 'b' -> sign * (int32 (Convert.ToUInt32(parseBinaryUInt64 s p l)))
    | 'o' -> sign * (int32 (Convert.ToUInt32(parseOctalUInt64 s p l)))
    | _ -> Int32.Parse(s, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture)
     
let lexemeTrimRightToInt32 args lexbuf n = 
    try parseInt32 (lexemeTrimRight lexbuf n)
    with _ -> fail args lexbuf (FSComp.SR.lexOutsideIntegerRange()) 0

//--------------------------
// Checks

let checkExprOp (lexbuf:UnicodeLexing.Lexbuf) = 
    if String.contains (lexeme lexbuf)  ':' then 
        deprecatedWithError (FSComp.SR.lexCharNotAllowedInOperatorNames(":")) lexbuf.LexemeRange
    if String.contains (lexeme lexbuf)  '$' then 
        deprecatedWithError (FSComp.SR.lexCharNotAllowedInOperatorNames("$")) lexbuf.LexemeRange 

let unexpectedChar lexbuf =
    LEX_FAILURE (FSComp.SR.lexUnexpectedChar(lexeme lexbuf))

let startString args (lexbuf: UnicodeLexing.Lexbuf) =
    let buf = ByteBuffer.Create 100 
    let m = lexbuf.LexemeRange 
    let startp = lexbuf.StartPos 
    let fin = (fun _m2 b s -> 
                     // Adjust the start-of-token mark back to the true start of the token 
                     lexbuf.StartPos <- startp
                     if b then
                        if Lexhelp.stringBufferIsBytes buf then
                            BYTEARRAY (Lexhelp.stringBufferAsBytes buf)
                        else (
                            fail args lexbuf (FSComp.SR.lexByteArrayCannotEncode()) ()
                            BYTEARRAY (Lexhelp.stringBufferAsBytes buf)
                        )
                     else
                        STRING (Lexhelp.stringBufferAsString s))  
    buf,fin,m
             
// Utility functions for processing XML documentation 

let trySaveXmlDoc lexbuf (buff:option<System.Text.StringBuilder>) =
    match buff with 
    | None -> () 
    | Some sb -> LexbufLocalXmlDocStore.SaveXmlDocLine (lexbuf, sb.ToString(), posOfLexPosition lexbuf.StartPos)
  
let tryAppendXmlDoc (buff:option<System.Text.StringBuilder>) (s:string) =
    match buff with 
    | None -> ()
    | Some sb -> ignore(sb.Append s)

// Utilities for parsing #if/#else/#endif 

let shouldStartLine args lexbuf (m:range) err tok = 
    if (m.StartColumn <> 0) then fail args lexbuf err tok
    else tok
    
let shouldStartFile args lexbuf (m:range) err tok = 
    if (m.StartColumn <> 0 || m.StartLine <> 1) then fail args lexbuf err tok
    else tok
    
let evalIfDefExpression startPos args (lookup:string->bool) (lexed:string) =
    let lexbuf          = LexBuffer<char>.FromChars (lexed.ToCharArray ())
    lexbuf.StartPos     <- startPos
    lexbuf.EndPos       <- startPos
    let tokenStream     = Microsoft.FSharp.Compiler.PPLexer.tokenstream args

    let expr            = Microsoft.FSharp.Compiler.PPParser.start tokenStream lexbuf

    LexerIfdefEval lookup expr

}
let letter = '\Lu' | '\Ll' | '\Lt' | '\Lm' | '\Lo' | '\Nl'
let surrogateChar = '\Cs'
let digit = '\Nd'
let hex = ['0'-'9'] | ['A'-'F'] | ['a'-'f']
let truewhite = [' ']
let offwhite = ['\t']
let anywhite = truewhite | offwhite
let anychar = [^'\n''\r']
let anystring = anychar*
let op_char = '!'|'$'|'%'|'&'|'*'|'+'|'-'|'.'|'/'|'<'|'='|'>'|'?'|'@'|'^'|'|'|'~'|':'
let ignored_op_char = '.' | '$' | '?'
let separator = '_'
let xinteger = 
  (  '0' ('x'| 'X')  hex ((hex | separator)* hex)?
   | '0' ('o'| 'O')  (['0'-'7']) (((['0'-'7']) | separator)* (['0'-'7']))?
   | '0' ('b'| 'B')  (['0'-'1']) (((['0'-'1']) | separator)* (['0'-'1']))?)
let integer = digit ((digit | separator)* digit)?
let int8 = integer 'y'
let uint8 = (xinteger | integer) 'u' 'y' 
let int16 = integer 's'
let uint16 = (xinteger | integer) 'u' 's'
let int = integer 
let int32 = integer 'l'
let uint32 = (xinteger | integer) 'u' 
let uint32l = (xinteger | integer) 'u' 'l'
let nativeint = (xinteger | integer) 'n'
let unativeint = (xinteger | integer) 'u' 'n'
let int64 = (xinteger | integer) 'L' 
let uint64 = (xinteger | integer) ('u' | 'U') 'L' 
let xint8 = xinteger 'y'
let xint16 = xinteger 's'
let xint = xinteger 
let xint32 = xinteger 'l'
let floatp = digit ((digit | separator)* digit)? '.' (digit ((digit | separator)* digit)?)?  
let floate = digit ((digit | separator)* digit)? ('.' (digit ((digit | separator)* digit)?)? )? ('e'| 'E') ['+' '-']? digit ((digit | separator)* digit)?
let float = floatp | floate 
let bignum =  integer ('I'  | 'N' | 'Z' | 'Q' | 'R' | 'G')
let ieee64 = float
let ieee32 = float ('f' | 'F') 
let decimal = (float | integer) ('m' | 'M') 
let xieee32 = xinteger 'l' 'f'
let xieee64 = xinteger 'L' 'F'
let escape_char = ('\\' ( '\\' | "\"" | '\'' | 'a' | 'f' | 'v' | 'n' | 't' | 'b' | 'r'))
let char = '\'' ( [^'\\''\n''\r''\t''\b'] | escape_char) '\''
let trigraph = '\\' digit digit digit
let hexGraphShort = '\\' 'x' hex hex 
let unicodeGraphShort = '\\' 'u' hex hex hex hex
let unicodeGraphLong =  '\\' 'U' hex hex hex hex hex hex hex hex
let newline = ('\n' | '\r' '\n')

let connecting_char = '\Pc'
let combining_char = '\Mn' | '\Mc'
let formatting_char = '\Cf' 

let ident_start_char = 
    letter | '_'

let ident_char = 
    letter
  | connecting_char 
  | combining_char 
  | formatting_char 
  | digit 
  | ['\'']
  
let ident = ident_start_char ident_char*

rule token args skip = parse
 | ident 
     { Keywords.KeywordOrIdentifierToken args lexbuf (lexeme lexbuf) }
 | "do!" 
     { DO_BANG } 
 | "yield!" 
     { YIELD_BANG(true)  } 
 | "return!" 
     { YIELD_BANG(false) } 
 | "match!"
     { MATCH_BANG }
 | ident '!' 
     { let tok = Keywords.KeywordOrIdentifierToken args lexbuf (lexemeTrimRight lexbuf 1) 
       match tok with 
       | LET _ -> BINDER (lexemeTrimRight lexbuf 1) 
       | _ -> fail args lexbuf (FSComp.SR.lexIdentEndInMarkReserved("!")) (Keywords.KeywordOrIdentifierToken args lexbuf (lexeme lexbuf)) } 
 | ident ('#')  
     { fail args lexbuf (FSComp.SR.lexIdentEndInMarkReserved("#")) (Keywords.KeywordOrIdentifierToken args lexbuf (lexeme lexbuf)) }
 | int8 
     { let n = lexemeTrimRightToInt32 args lexbuf 1
       if n > 0x80 || n < -0x80 then fail args lexbuf (FSComp.SR.lexOutsideEightBitSigned()) (INT8(0y,false))
       // Allow <max_int+1> to parse as min_int.  Allowed only because we parse '-' as an operator. 
       else if n = 0x80 then INT8(sbyte(-0x80), true (* 'true' = 'bad'*) )
       else INT8(sbyte n,false) }
 | xint8 
     { let n = lexemeTrimRightToInt32 args lexbuf 1
       if n > 0xFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitSignedHex()) (INT8(0y,false))
       else INT8(sbyte(byte(n)),false) }
 | uint8
     { let n = lexemeTrimRightToInt32 args lexbuf 2
       if n > 0xFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitUnsigned()) (UINT8(0uy))
       else UINT8(byte n) }
 | int16 
     { let n = lexemeTrimRightToInt32 args lexbuf 1
       if n > 0x8000 || n < -0x8000 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s,false))
       // Allow <max_int+1> to parse as min_int.  Allowed only because we parse '-' as an operator. 
       else if n = 0x8000 then INT16(-0x8000s,true)
       else INT16(int16 n,false) }
 | xint16 
     { let n = lexemeTrimRightToInt32 args lexbuf 1
       if n > 0xFFFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s,false))
       else INT16(int16(uint16(n)),false) }
 | uint16 
     { let n = lexemeTrimRightToInt32 args lexbuf 2
       if n > 0xFFFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitUnsigned()) (UINT16(0us))
       else UINT16(uint16 n) }
 | int '.' '.' 
     { let s = removeUnderscores (lexemeTrimRight lexbuf 2)
       // Allow <max_int+1> to parse as min_int.  Allowed only because we parse '-' as an operator. 
       if s = "2147483648" then INT32_DOT_DOT(-2147483648,true) else
       let n = try int32 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
       INT32_DOT_DOT(n,false)
     } 
 | xint 
 | int 
     { let s = removeUnderscores (lexeme lexbuf)
       // Allow <max_int+1> to parse as min_int.  Allowed only because we parse '-' as an operator. 
       if s = "2147483648" then INT32(-2147483648,true) else
       let n =
           try int32 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
       INT32(n,false)
     } 
 | xint32 
 | int32 
     { let s = removeUnderscores (lexemeTrimRight lexbuf 1)
       // Allow <max_int+1> to parse as min_int.  Allowed only because we parse '-' as an operator. 
       if s = "2147483648" then INT32(-2147483648,true) else
       let n = 
           try int32 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
       INT32(n,false)
     } 

 | uint32
     { 
       let s = removeUnderscores (lexemeTrimRight lexbuf 1)
       let n = 
           try int64 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) 0L
       if n > 0xFFFFFFFFL || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
       UINT32(uint32 (uint64 n)) } 

 | uint32l
     { 
       let s = removeUnderscores (lexemeTrimRight lexbuf 2)
       let n = 
           try int64 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) 0L
       if n > 0xFFFFFFFFL || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
       UINT32(uint32 (uint64 n)) } 

 | int64 
     { let s = removeUnderscores (lexemeTrimRight lexbuf 1)
       // Allow <max_int+1> to parse as min_int.  Stupid but allowed because we parse '-' as an operator. 
       if s = "9223372036854775808" then INT64(-9223372036854775808L,true) else
       let n = 
          try int64 s with _ ->  fail args lexbuf (FSComp.SR.lexOutsideSixtyFourBitSigned()) 0L
       INT64(n,false)
     }

 | uint64     
     { let s = removeUnderscores (lexemeTrimRight lexbuf 2)
       let n = 
         try uint64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideSixtyFourBitUnsigned()) 0UL
       UINT64(n) } 

 | nativeint  
     { try 
           NATIVEINT(int64 (removeUnderscores (lexemeTrimRight lexbuf 1)))
       with _ ->  fail args lexbuf (FSComp.SR.lexOutsideNativeSigned()) (NATIVEINT(0L)) } 

 | unativeint 
     { try 
           UNATIVEINT(uint64 (removeUnderscores (lexemeTrimRight lexbuf 2)))
       with _ ->  fail args lexbuf (FSComp.SR.lexOutsideNativeUnsigned())  (UNATIVEINT(0UL)) }

 | ieee32     
     { IEEE32 (try float32(removeUnderscores (lexemeTrimRight lexbuf 1)) with _ -> fail args lexbuf (FSComp.SR.lexInvalidFloat()) 0.0f) }
 | ieee64     
     { IEEE64 (try float(lexeme lexbuf) with _ -> fail args lexbuf (FSComp.SR.lexInvalidFloat()) 0.0) }

 | decimal    
     { try 
          let s = removeUnderscores (lexemeTrimRight lexbuf 1)
          // This implements a range check for decimal literals 
          let d = System.Decimal.Parse(s,System.Globalization.NumberStyles.AllowExponent ||| System.Globalization.NumberStyles.Number,System.Globalization.CultureInfo.InvariantCulture)
          DECIMAL d 
       with 
          e -> fail args lexbuf (FSComp.SR.lexOusideDecimal()) (DECIMAL (decimal 0))
     }
 | xieee32     
     { 
       let s = removeUnderscores (lexemeTrimRight lexbuf 2)
       // Even though the intermediate step is an int64, display the "invalid float" message, since it will be less confusing to the user 
       let n64 = (try (int64 s) with _ -> fail args lexbuf (FSComp.SR.lexInvalidFloat()) 0L) 
       if n64 > 0xFFFFFFFFL || n64 < 0L then fail args lexbuf (FSComp.SR.lexOusideThirtyTwoBitFloat()) (IEEE32 0.0f) else
       IEEE32 (System.BitConverter.ToSingle(System.BitConverter.GetBytes(int32 (uint32 (uint64 n64))),0)) }

 | xieee64     
     { 
       let n64 = (try int64 (removeUnderscores (lexemeTrimRight lexbuf 2)) with _ -> fail args lexbuf (FSComp.SR.lexInvalidFloat()) 0L) 
       IEEE64 (System.BitConverter.Int64BitsToDouble(n64)) }
       
 | bignum     
       { let s = lexeme lexbuf 
         BIGNUM (removeUnderscores (lexemeTrimRight lexbuf 1), s.[s.Length-1..s.Length-1]) }

 | (int | xint | float) ident_char+
       { fail args lexbuf (FSComp.SR.lexInvalidNumericLiteral()) (INT32(0,false)) }
 
 | char
     { let s = lexeme lexbuf 
       CHAR (if s.[1] = '\\' then escape s.[2] else s.[1])  }

 | char 'B' 
     { let s = lexeme lexbuf 
       let x = int32 (if s.[1] = '\\' then escape s.[2] else s.[1])
       if x < 0 || x > 127 then 
           fail args lexbuf (FSComp.SR.lexInvalidByteLiteral()) (UINT8(byte 0))
       else
           UINT8 (byte(x))  }
     
 | '\'' trigraph '\''
     { let s = lexeme lexbuf 
       let c = trigraph s.[2] s.[3] s.[4] 
       let x = int32 c
       if x < 0 || x > 255 then 
           fail args lexbuf (FSComp.SR.lexInvalidCharLiteral()) (CHAR c)
       else
           CHAR c }

 | '\'' trigraph '\'' 'B'
     { let s = lexeme lexbuf 
       let x = int32 (trigraph s.[2] s.[3] s.[4])
       if x < 0 || x > 255 then 
           fail args lexbuf (FSComp.SR.lexInvalidByteLiteral()) (UINT8(byte 0))
       else
           UINT8 (byte(x))  }

 | '\'' unicodeGraphShort '\''  'B'
     { let x = int32 (unicodeGraphShort (lexemeTrimBoth lexbuf 3 2))
       if x < 0 || x > 127 then 
           fail args lexbuf (FSComp.SR.lexInvalidByteLiteral()) (UINT8(byte 0))
       else
           UINT8 (byte(x))  }
     
 | '\'' hexGraphShort '\'' { CHAR (char (int32 (hexGraphShort (lexemeTrimBoth lexbuf 3 1)))) }
 | '\'' unicodeGraphShort '\'' { CHAR (char (int32 (unicodeGraphShort (lexemeTrimBoth lexbuf 3 1)))) }
 | '\'' unicodeGraphLong '\''  
     { match unicodeGraphLong (lexemeTrimBoth lexbuf 3 1) with
       | SingleChar(c) -> CHAR (char c)
       | _ -> fail args lexbuf  (FSComp.SR.lexThisUnicodeOnlyInStringLiterals()) (CHAR (char 0)) }
 | "(*IF-FSHARP"    
     { if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
 | "(*F#"            
     { if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
 | "ENDIF-FSHARP*)" 
     { if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf  }
 | "F#*)"           
     { if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }

 | "(*)"            
     { LPAREN_STAR_RPAREN }

 | "(*"
     { let m = lexbuf.LexemeRange 
       if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,1,m))) else comment (1,m,args) skip lexbuf }

 | "(*IF-CAML*)" |  "(*IF-OCAML*)" 
     { let m = lexbuf.LexemeRange 
       if not skip then (COMMENT (LexCont.MLOnly(!args.ifdefStack,m))) else mlOnly m args skip lexbuf }

 | '"' 
     { let buf,fin,m = startString args lexbuf 
       if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m))) else string (buf,fin,m,args) skip lexbuf }
      
 | '"' '"' '"' 
     { let buf,fin,m = startString args lexbuf 
       if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m))) else tripleQuoteString (buf,fin,m,args) skip lexbuf }
      
 | '$' '"' 
     { fail args lexbuf  (FSComp.SR.lexTokenReserved()) (WHITESPACE (LexCont.Token !args.ifdefStack)) }
      
 | '@' '"' 
     { let buf,fin,m = startString args lexbuf 
       if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m))) else verbatimString (buf,fin,m,args) skip lexbuf }

 | truewhite+  
     { if skip then token args skip lexbuf
       else WHITESPACE (LexCont.Token !args.ifdefStack) }

 | offwhite+  
     { if args.lightSyntaxStatus.Status then errorR(Error(FSComp.SR.lexTabsNotAllowed(),lexbuf.LexemeRange))
       if not skip then (WHITESPACE (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }

 | "////" op_char* 
     { // 4+ slash are 1-line comments, online 3 slash are XmlDoc 
       let m = lexbuf.LexemeRange 
       if not skip then (LINE_COMMENT (LexCont.SingleLineComment(!args.ifdefStack,1,m))) else singleLineComment (None,1,m,args) skip lexbuf }

 | "///" op_char*
     { // Match exactly 3 slash, 4+ slash caught by preceding rule 
       let m = lexbuf.LexemeRange
       let doc = lexemeTrimLeft lexbuf 3  
       let sb = (new StringBuilder(100)).Append(doc)
       if not skip then (LINE_COMMENT (LexCont.SingleLineComment(!args.ifdefStack,1,m))) else singleLineComment (Some sb,1,m,args) skip lexbuf }

 | "//" op_char*
     { // Need to read all operator symbols too, otherwise it might be parsed by a rule below 
       let m = lexbuf.LexemeRange
       if not skip then (LINE_COMMENT (LexCont.SingleLineComment(!args.ifdefStack,1,m))) else singleLineComment (None,1,m,args) skip lexbuf }

 | newline 
     { newline lexbuf; if not skip then (WHITESPACE (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }

 | '`' '`' ([^'`' '\n' '\r' '\t'] | '`' [^'`''\n' '\r' '\t']) + '`' '`' 
     { Keywords.IdentifierToken args lexbuf (lexemeTrimBoth lexbuf 2 2) }

 | ('#' anywhite* | "#line" anywhite+ ) digit+ anywhite* ('@'? "\"" [^'\n''\r''"']+ '"')? anywhite* newline
     {  let pos = lexbuf.EndPos 
        if skip then 
          let s = lexeme lexbuf 
          let rec parseLeadingDirective n = 
            match s.[n] with 
            | c when c >= 'a' && c <= 'z' -> parseLeadingDirective (n+1) 
            | _ -> parseLeadingWhitespace n // goto the next state
          
          and parseLeadingWhitespace n = 
            match s.[n] with 
            | ' ' | '\t' -> parseLeadingWhitespace (n+1) 
            | _ -> parseLineNumber n n // goto the next state
          
          and parseLineNumber start n = 
            match s.[n] with 
            | c when c >= '0' && c <= '9' -> parseLineNumber start (n+1)
            | _ -> let text =  (String.sub s start (n-start)) 
                   let lineNumber = 
                       try int32 text
                       with err -> errorR(Error(FSComp.SR.lexInvalidLineNumber(text),lexbuf.LexemeRange)); 0
                   lineNumber, parseWhitespaceBeforeFile n // goto the next state
          
          and parseWhitespaceBeforeFile n =  
            match s.[n] with 
            | ' ' | '\t' | '@' -> parseWhitespaceBeforeFile (n+1) 
            | '"' -> Some (parseFile (n+1) (n+1))
            | _ -> None
          
          and parseFile start n =   
            match s.[n] with 
            | '"' -> String.sub s start (n-start)  
            | _ -> parseFile start (n+1) 

          // Call the parser
          let line,file = parseLeadingDirective 1 

          // Construct the new position
          if args.applyLineDirectives then 
              lexbuf.EndPos <- pos.ApplyLineDirective((match file with Some f -> fileIndexOfFile f | None -> pos.FileIndex), line)

          token args skip lexbuf 
        else 
          if not skip then (HASH_LINE (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
            
 | "<@" { checkExprOp lexbuf; LQUOTE ("<@ @>", false) }
 | "<@@" { checkExprOp lexbuf; LQUOTE ("<@@ @@>", true) }
 | "@>" { checkExprOp lexbuf; RQUOTE ("<@ @>", false) }
 | "@@>" { checkExprOp lexbuf; RQUOTE ("<@@ @@>", true) }
 | '#' { HASH }
 | '&' { AMP }
 | "&&" { AMP_AMP }
 | "||" { BAR_BAR }
 | '\'' { QUOTE }
 | '(' { LPAREN }
 | ')' { RPAREN }
 | '*' { STAR }
 | ',' { COMMA }
 | "->" { RARROW }
 | "?" { QMARK }
 | "??" { QMARK_QMARK }
 | ".." { DOT_DOT }
 | "." { DOT }
 | ":" { COLON }
 | "::" { COLON_COLON }
 | ":>" { COLON_GREATER }
 | "@>." { RQUOTE_DOT ("<@ @>",false) }
 | "@@>." { RQUOTE_DOT ("<@@ @@>",true) }
 | ">|]" { GREATER_BAR_RBRACK }
 | ":?>" { COLON_QMARK_GREATER }
 | ":?" { COLON_QMARK }
 | ":=" { COLON_EQUALS }
 | ";;" { SEMICOLON_SEMICOLON }
 | ";" { SEMICOLON }
 | "<-" { LARROW }
 | "=" { EQUALS }
 | "[" { LBRACK }
 | "[|" { LBRACK_BAR }
 | "<" { LESS false }
 | ">" { GREATER false }
 | "[<" { LBRACK_LESS }
 | "]" { RBRACK }
 | "|]" { BAR_RBRACK }
 | ">]" { GREATER_RBRACK }
 | "{" { LBRACE }
 | "|" { BAR }
 | "}" { RBRACE }
 | "$" { DOLLAR }
 | "%" { PERCENT_OP("%") }
 | "%%" { PERCENT_OP("%%") }
 | "-" { MINUS }
 | "~" { RESERVED }
 | "`" { RESERVED }
 | ignored_op_char* '*' '*'                    op_char* { checkExprOp lexbuf; INFIX_STAR_STAR_OP(lexeme lexbuf) }
 | ignored_op_char* ('*' | '/'|'%')            op_char* { checkExprOp lexbuf; INFIX_STAR_DIV_MOD_OP(lexeme lexbuf) }
 | ignored_op_char* ('+'|'-')                  op_char* { checkExprOp lexbuf; PLUS_MINUS_OP(lexeme lexbuf) }
 | ignored_op_char* ('@'|'^')                  op_char* { checkExprOp lexbuf; INFIX_AT_HAT_OP(lexeme lexbuf) }
 | ignored_op_char* ('=' | "!=" | '<' | '>' | '$')  op_char* { checkExprOp lexbuf; INFIX_COMPARE_OP(lexeme lexbuf) }
 | ignored_op_char* ('&')                      op_char* { checkExprOp lexbuf; INFIX_AMP_OP(lexeme lexbuf) }
 | ignored_op_char* '|'                        op_char* { checkExprOp lexbuf; INFIX_BAR_OP(lexeme lexbuf) }
 | ignored_op_char* ('!' | '~' )               op_char* { checkExprOp lexbuf; PREFIX_OP(lexeme lexbuf) }
 | ".[]"  | ".[]<-" | ".[,]<-" | ".[,,]<-" | ".[,,,]<-" | ".[,,,]" | ".[,,]" | ".[,]" | ".[..]" | ".[..,..]" | ".[..,..,..]" | ".[..,..,..,..]"
 | ".()"  | ".()<-"  { FUNKY_OPERATOR_NAME(lexeme lexbuf) }
 | "#!" op_char* 
   { // Treat shebangs like regular comments, but they are only allowed at the start of a file
       let m = lexbuf.LexemeRange
       let tok = shouldStartFile args lexbuf m (0,FSComp.SR.lexHashBangMustBeFirstInFile()) (LINE_COMMENT (LexCont.SingleLineComment(!args.ifdefStack,1,m)))
       if not skip then tok else singleLineComment (None,1,m,args) skip lexbuf }

 | "#light" anywhite* 
 | ("#indent" | "#light") anywhite+ "\"on\""
   { if args.lightSyntaxStatus.ExplicitlySet && args.lightSyntaxStatus.WarnOnMultipleTokens then 
         warning(Error((0,"#light should only occur as the first non-comment text in an F# source file"),lexbuf.LexemeRange))
     // TODO unreachable error above, I think? - brianmcn 
     args.lightSyntaxStatus.Status <- true 
     if not skip then (HASH_LIGHT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf } 

 | ("#indent" | "#light") anywhite+ "\"off\"" 
   { args.lightSyntaxStatus.Status <- false 
     mlCompatWarning (FSComp.SR.lexIndentOffForML()) lexbuf.LexemeRange
     if not skip then (HASH_LIGHT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf } 
   
 | anywhite* "#if" anywhite+ anystring
   { let m = lexbuf.LexemeRange   
     let lookup id = List.contains id args.defines
     let lexed = lexeme lexbuf
     let isTrue = evalIfDefExpression lexbuf.StartPos args lookup lexed
     args.ifdefStack := (IfDefIf,m) :: !(args.ifdefStack)
     
     // Get the token; make sure it starts at zero position & return
     let cont, f = 
       ( if isTrue then (LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)), endline (LexerEndlineContinuation.Token !args.ifdefStack) args skip)
         else (LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)), endline (LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)) args skip) )         
     let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) (HASH_IF(m,lexed,cont))
     if not skip then tok else f lexbuf }

 | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)?
   { let lexed = (lexeme lexbuf)
     match !(args.ifdefStack) with
     | [] ->  LEX_FAILURE (FSComp.SR.lexHashElseNoMatchingIf())
     | (IfDefElse,_) :: _rest -> LEX_FAILURE (FSComp.SR.lexHashEndifRequiredForElse())
     | (IfDefIf,_) :: rest -> 
       let m = lexbuf.LexemeRange 
       args.ifdefStack := (IfDefElse,m) :: rest
       let tok = HASH_ELSE(m,lexed, LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)))
       let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) tok
       if not skip then tok else endline (LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)) args skip lexbuf }

 | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)?
   { let lexed = (lexeme lexbuf) 
     let m = lexbuf.LexemeRange 
     match !(args.ifdefStack) with
     | []->  LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf())
     | _ :: rest ->  
        args.ifdefStack := rest          
        let tok = HASH_ENDIF(m,lexed,LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack))) 
        let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) tok 
        if not skip then tok else endline (LexerEndlineContinuation.Token(!args.ifdefStack)) args skip lexbuf }

 | "#if" 
   { let tok = fail args lexbuf (FSComp.SR.lexHashIfMustHaveIdent()) (WHITESPACE (LexCont.Token !args.ifdefStack)) 
     if not skip then tok else token args skip lexbuf }

 | surrogateChar surrogateChar
 | _ 
   { unexpectedChar lexbuf }     
 | eof 
   { EOF (LexCont.Token !args.ifdefStack) }

// Skips INACTIVE code until if finds #else / #endif matching with the #if or #else 

and ifdefSkip n m args skip = parse               
 | anywhite* "#if" anywhite+ anystring
   { let m = lexbuf.LexemeRange    
     
     // If #if is the first thing on the line then increase depth, otherwise skip, because it is invalid (e.g. "(**) #if ...")
     if (m.StartColumn <> 0) then
       if not skip then (INACTIVECODE (LexCont.IfDefSkip(!args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf
     else
       let tok = INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,n+1,m))) 
       if not skip then tok else endline (LexerEndlineContinuation.Skip(!args.ifdefStack,n+1,m)) args skip lexbuf }

  | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)?
    { let lexed = (lexeme lexbuf) 
      let m = lexbuf.LexemeRange  
           
      // If #else is the first thing on the line then process it, otherwise ignore, because it is invalid (e.g. "(**) #else ...")
      if (m.StartColumn <> 0) then
        if not skip then (INACTIVECODE (LexCont.IfDefSkip(!args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf
      elif n = 0 then 
         match !(args.ifdefStack) with
         | []->  LEX_FAILURE (FSComp.SR.lexHashElseNoMatchingIf())
         | (IfDefElse,_) :: _rest -> LEX_FAILURE (FSComp.SR.lexHashEndifRequiredForElse())
         | (IfDefIf,_) :: rest -> 
           let m = lexbuf.LexemeRange  
           args.ifdefStack := (IfDefElse,m) :: rest
           if not skip then (HASH_ELSE(m,lexed,LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)))) else endline (LexerEndlineContinuation.Token(!args.ifdefStack)) args skip lexbuf 
       else
         if not skip then (INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,n,m)))) else endline (LexerEndlineContinuation.Skip(!args.ifdefStack,n,m)) args skip lexbuf }
          
  | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)?
    { let lexed = lexeme lexbuf
      let m = lexbuf.LexemeRange  
      
      // If #endif is the first thing on the line then process it, otherwise ignore, because it is invalid (e.g. "(**) #endif ...")
      if (m.StartColumn <> 0) then
          if not skip then (INACTIVECODE (LexCont.IfDefSkip(!args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf
      elif n = 0 then 
          match !(args.ifdefStack) with
          | [] ->  LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf())
          | _ :: rest -> 
              args.ifdefStack := rest
              if not skip then (HASH_ENDIF(m,lexed,LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)))) else endline (LexerEndlineContinuation.Token(!args.ifdefStack)) args skip lexbuf 
       else
           let tok = INACTIVECODE(LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,n-1,m)))
           let tok = shouldStartLine args lexbuf m (FSComp.SR.lexWrongNestedHashEndif()) tok
           if not skip then tok else endline (LexerEndlineContinuation.Skip(!args.ifdefStack,(n-1),m)) args skip lexbuf }
           
  | newline 
    { newline lexbuf; ifdefSkip n m args skip lexbuf }
    
  | [^ ' ' '\n' '\r' ]+
  | anywhite+
  | surrogateChar surrogateChar
  | _    
    { // This tries to be nice and get tokens as 'words' because VS uses this when selecting stuff
      if not skip then (INACTIVECODE (LexCont.IfDefSkip(!args.ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf }
  | eof  
    { EOF (LexCont.IfDefSkip(!args.ifdefStack,n,m)) }

// Called after lexing #if IDENT/#else/#endif - this checks whether there is nothing except end of line 
// or end of file and then calls the lexing function specified by 'cont' - either token or ifdefSkip 
and endline cont args skip = parse
 | newline
   { newline lexbuf 
     match cont with
     | LexerEndlineContinuation.Token(ifdefStack) -> if not skip then (WHITESPACE(LexCont.Token ifdefStack)) else token args skip lexbuf
     | LexerEndlineContinuation.Skip(ifdefStack, n, m) -> if not skip then (INACTIVECODE (LexCont.IfDefSkip(ifdefStack,n,m))) else ifdefSkip n m args skip lexbuf
   }
 | eof
   { match cont with
     | LexerEndlineContinuation.Token(ifdefStack) -> (EOF(LexCont.Token ifdefStack))
     | LexerEndlineContinuation.Skip(ifdefStack, n, m) -> (EOF(LexCont.IfDefSkip(ifdefStack,n,m)))
   }
 | [^'\r' '\n']+
 | _
   { let tok = fail args lexbuf (FSComp.SR.pplexExpectedSingleLineComment()) (WHITESPACE (LexCont.Token !args.ifdefStack))
     if not skip then tok else token args skip lexbuf }     

and string sargs skip = parse
 |  '\\' newline anywhite* 
    { let (_buf,_fin,m,args) = sargs 
      newline lexbuf 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 |  escape_char
    { let (buf,_fin,m,args) = sargs 
      addByteChar buf (escape (lexeme lexbuf).[1])
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf } 

 | trigraph
    { let (buf,_fin,m,args) = sargs 
      let s = lexeme lexbuf 
      addByteChar buf (trigraph s.[1] s.[2] s.[3])
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 | hexGraphShort
    { let (buf,_fin,m,args) = sargs 
      addUnicodeChar buf (int (hexGraphShort (lexemeTrimLeft lexbuf 2)))
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf  }
      
 | unicodeGraphShort
    { let (buf,_fin,m,args) = sargs 
      addUnicodeChar buf (int (unicodeGraphShort (lexemeTrimLeft lexbuf 2)))
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf  }
     
 | unicodeGraphLong
    { let (buf,_fin,m,args) = sargs
      let hexChars = lexemeTrimLeft lexbuf 2
      let result () = if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m))) else string sargs skip lexbuf
      match unicodeGraphLong hexChars with
      | Invalid ->
          fail args lexbuf (FSComp.SR.lexInvalidUnicodeLiteral hexChars) (result ())
      | SingleChar(c) ->
          addUnicodeChar buf (int c)
          result ()
      | SurrogatePair(hi, lo) -> 
          addUnicodeChar buf (int hi)
          addUnicodeChar buf (int lo)
          result () }
     
 |  '"' 
    { let (buf,fin,_m,_args) = sargs 
      let m2 = lexbuf.LexemeRange  
      callStringFinisher fin buf m2 false }

 |  '"''B' 
    { let (buf,fin,_m,_args) = sargs 
      let m2 = lexbuf.LexemeRange 
      callStringFinisher fin buf m2 true }

 | newline
    { let (buf,_fin,m,args) = sargs 
      newline lexbuf 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 | ident  
    { let (buf,_fin,m,args) = sargs  
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 | integer 
 | xinteger
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 | anywhite +  
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m)))  else string sargs skip lexbuf }

 | eof  
    { let (_buf,_fin,m,args) = sargs 
      EOF (LexCont.String(!args.ifdefStack,m)) }
 | surrogateChar surrogateChar // surrogate code points always come in pairs      
 | _ 
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.String(!args.ifdefStack,m))) else string sargs skip lexbuf }

and verbatimString sargs skip = parse
 |  '"' '"'
   { let (buf,_fin,m,args) = sargs 
     addByteChar buf '\"'
     if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m)))  else verbatimString sargs skip lexbuf } 

 |  '"' 
    { let (buf,fin,_m,_args) = sargs 
      let m2 = lexbuf.LexemeRange 
      callStringFinisher fin buf m2 false }

 |  '"''B' 
    { let (buf,fin,_m,_args) = sargs 
      let m2 = lexbuf.LexemeRange 
      callStringFinisher fin buf m2 true }

 | newline 
    { let (buf,_fin,m,args) = sargs 
      newline lexbuf 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m)))  else verbatimString sargs skip lexbuf }

 | ident  
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m)))  else verbatimString sargs skip lexbuf }

 | integer 
 | xinteger
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m)))  else verbatimString sargs skip lexbuf }

 | anywhite +  
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m)))  else verbatimString sargs skip lexbuf }

 | eof 
    { let (_buf,_fin,m,args) = sargs 
      EOF (LexCont.VerbatimString(!args.ifdefStack,m)) }
 | surrogateChar surrogateChar // surrogate code points always come in pairs            
 | _ 
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.VerbatimString(!args.ifdefStack,m))) else verbatimString sargs skip lexbuf }

and tripleQuoteString sargs skip = parse
 |  '"' '"' '"' 
    { let (buf,fin,_m,_args) = sargs 
      let m2 = lexbuf.LexemeRange 
      callStringFinisher fin buf m2 false }

 | newline 
    { let (buf,_fin,m,args) = sargs 
      newline lexbuf 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m)))  else tripleQuoteString sargs skip lexbuf }

// The rest is to break into pieces to allow double-click-on-word and other such things 
 | ident  
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m)))  else tripleQuoteString sargs skip lexbuf }

 | integer 
 | xinteger
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m)))  else tripleQuoteString sargs skip lexbuf }

 | anywhite +  
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m)))  else tripleQuoteString sargs skip lexbuf }

 | eof 
    { let (_buf,_fin,m,args) = sargs 
      EOF (LexCont.TripleQuoteString(!args.ifdefStack,m)) }
 | surrogateChar surrogateChar // surrogate code points always come in pairs            
 | _ 
    { let (buf,_fin,m,args) = sargs 
      addUnicodeString buf (lexeme lexbuf) 
      if not skip then (STRING_TEXT (LexCont.TripleQuoteString(!args.ifdefStack,m))) else tripleQuoteString sargs skip lexbuf }

// Parsing single-line comment - we need to split it into words for Visual Studio IDE 
and singleLineComment cargs skip = parse 
 | newline
     { let buff,_n,_m,args = cargs 
       trySaveXmlDoc lexbuf buff
       newline lexbuf 
       // Saves the documentation (if we're collecting any) into a buffer-local variable.
       if not skip then (LINE_COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
 
 | eof 
     { let _, _n,_m,args = cargs 
       // NOTE: it is legal to end a file with this comment, so we'll return EOF as a token 
       EOF (LexCont.Token !args.ifdefStack) } 
      
 | [^ ' ' '\n' '\r' ]+
 | anywhite+
     { let buff,n,m,args = cargs 
       // Append the current token to the XML documentation if we're collecting it 
       tryAppendXmlDoc buff (lexeme lexbuf)
       if not skip then (LINE_COMMENT (LexCont.SingleLineComment(!args.ifdefStack,n,m))) else singleLineComment (buff,n,m,args) skip lexbuf  } 
       
 | surrogateChar surrogateChar
 | _ { let _, _n,_m,args = cargs 
      if not skip then (LINE_COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }

       
and comment cargs skip = parse
 |  char
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf  } 
    
 | '"'   
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.StringInComment(!args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf }

 | '"' '"' '"'  
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(!args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf }

 | '@' '"'
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf }

 | "(*)"
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment cargs skip lexbuf }

 | '(' '*'
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n+1,m))) else comment (n+1,m,args) skip lexbuf }
     
 | newline
    { let n,m,args = cargs 
      newline lexbuf 
      if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment cargs skip lexbuf }
 | "*)" 
    { 
      let n,m,args = cargs 
      if n > 1 then if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n-1,m))) else comment (n-1,m,args) skip lexbuf 
      else if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
      
 | anywhite+
 | [^ '\'' '(' '*' '\n' '\r' '"' ')' '@' ' ' '\t' ]+  
    { let n,m,args = cargs 
      if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment cargs skip lexbuf }
    
 | eof 
     { let n,m,args = cargs 
       EOF (LexCont.Comment(!args.ifdefStack,n,m)) }
     
 | surrogateChar surrogateChar
 | _ { let n,m,args = cargs 
       if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf }

and stringInComment n m args skip = parse
 // Follow string lexing, skipping tokens until it finishes 
 |  '\\' newline anywhite* 
     { newline lexbuf 
       if not skip then (COMMENT (LexCont.StringInComment(!args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf }

 | escape_char
 | trigraph
 | hexGraphShort
 | unicodeGraphShort
 | unicodeGraphLong
 | ident  
 | integer
 | xinteger
 | anywhite +  
     { if not skip then (COMMENT (LexCont.StringInComment(!args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf }


 | '"' 
     { if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf }
     
 | newline 
     { newline lexbuf  
       if not skip then (COMMENT (LexCont.StringInComment(!args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf }
     
 | eof 
     { EOF (LexCont.StringInComment(!args.ifdefStack,n,m)) }
     
 | surrogateChar surrogateChar
 | _  
     { if not skip then (COMMENT (LexCont.StringInComment(!args.ifdefStack,n,m))) else stringInComment n m args skip lexbuf }

and verbatimStringInComment n m args skip = parse
 // Follow verbatimString lexing, in short, skip double-quotes and other chars until we hit a single quote 
 | '"' '"'
     { if not skip then (COMMENT (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf }

 | '"' 
     { if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf }

 | ident  
 | integer 
 | xinteger
 | anywhite +  
     { if not skip then (COMMENT (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf }
     
 | newline 
     { newline lexbuf
       if not skip then (COMMENT (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf }
     
 | eof 
     { EOF (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m)) }
 
 | surrogateChar surrogateChar
 | _  
     { if not skip then (COMMENT (LexCont.VerbatimStringInComment(!args.ifdefStack,n,m))) else verbatimStringInComment n m args skip lexbuf }
     
and tripleQuoteStringInComment n m args skip = parse
 // Follow tripleQuoteString lexing 
 | '"' '"' '"'
     { if not skip then (COMMENT (LexCont.Comment(!args.ifdefStack,n,m))) else comment (n,m,args) skip lexbuf }

 | ident  
 | integer 
 | xinteger
 | anywhite +  
     { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(!args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf }
     
 | newline 
     { newline lexbuf
       if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(!args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf }
     
 | eof 
     { EOF (LexCont.TripleQuoteStringInComment(!args.ifdefStack,n,m)) }
 
 | surrogateChar surrogateChar
 | _  
     { if not skip then (COMMENT (LexCont.TripleQuoteStringInComment(!args.ifdefStack,n,m))) else tripleQuoteStringInComment n m args skip lexbuf }
     

and mlOnly m args skip = parse
 | "\""
     { let buf = ByteBuffer.Create 100 
       let m2 = lexbuf.LexemeRange 
       let _ = string (buf,defaultStringFinisher,m2,args) skip lexbuf 
       if not skip then (COMMENT (LexCont.MLOnly(!args.ifdefStack,m))) else mlOnly m args skip lexbuf }
 | newline 
     { newline lexbuf;  if not skip then (COMMENT (LexCont.MLOnly(!args.ifdefStack,m))) else mlOnly m args skip lexbuf }
 | "(*ENDIF-CAML*)"  
     {  if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
 | "(*ENDIF-OCAML*)" 
     {  if not skip then (COMMENT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf }
 | [^ '(' '"' '\n' '\r' ]+  
     { if not skip then (COMMENT (LexCont.MLOnly(!args.ifdefStack,m))) else mlOnly m args skip lexbuf }
 | eof 
     { EOF (LexCont.MLOnly(!args.ifdefStack,m)) }
 | surrogateChar surrogateChar 
 | _ 
     {  if not skip then (COMMENT (LexCont.MLOnly(!args.ifdefStack,m))) else mlOnly m args skip lexbuf }