/*
 * Copyright (c) 2003 - 2006, Nils R. Weller
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Helper functions for parsing string and character constants,
 * trigraphs, operators and constants. Also contains global token
 * list and functions to add to this list. Used by lexical
 * analyzer.
 */
#include "token.h"
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <ctype.h>
#include "error.h"
#include "defs.h"
#include "numlimits.h"
#include "preprocess.h"
#include "type.h"
#include "n_libc.h"

size_t		lex_chars_read;
char		*lex_line_ptr;
char		*lex_tok_ptr;
char		*lex_file_map;
char		*lex_file_map_end;
extern int	g_recording_tokens;


struct ty_string		*str_const = NULL;

struct ty_float			*float_const = NULL;

struct token *
alloc_token(void) {
	struct token	*ret = n_xmalloc(sizeof *ret);
	static struct token	nulltok;
	*ret = nulltok;
	return ret;
}

void
free_tokens(struct token *from, struct token *to, int type) {
	do {
		struct token	*next = from->next;
		
		if (type == FREE_DECL) {
			if (from->type == TOK_PAREN_OPEN
				|| from->type == TOK_PAREN_CLOSE
				|| from->type == TOK_ARRAY_OPEN
				|| from->type == TOK_ARRAY_CLOSE
				|| from->type == TOK_SEMICOLON
				|| IS_KEYWORD(from->type)) {
				free(from);
			} else if (from->type == TOK_OPERATOR
				&& *(int *)from->data == TOK_OP_ASSIGN) {
				break;
			}	
		} else {
			abort();
		}	
		from = next;
	} while (from != to);
}


#if 0
static int
get_next_char(FILE *fd) {
	int	ch = getc(fd);
	if (ch == '\n') {
		lex_line_ptr = lex_file_map + lex_chars_read;
		err_setlineptr(lex_line_ptr);
	}
	return ch;
}
#endif

/*
 * Routine used by several functions in this module - stores
 * character ch in *p, resizing it if necessary. Must be called
 * with a first argument of NULL before it can be used in order
 * to initialize static data. Exits if no memory is available.
 *
 * XXX Not reentrant
 * XXX Should use nwstr library instead, which may also speed up
 * detecing keywords in the source
 */
int 
store_char(char **p, int ch) {
	static int	chunksiz	= 32;
	static int	alloc		= 0;
	static int	index		= 0;

	if (p == NULL) {
		 /* Initialize static data */
		 chunksiz = 8;
		 alloc = 0;
		 index= 0;
		 return 0;
	}

	if (index >= alloc) {
		alloc += chunksiz;
		chunksiz *= 2;
		if ((*p = n_xrealloc(*p, alloc)) == NULL) {
			perror("realloc");
			exit(EXIT_FAILURE);
		}
	}	
	(*p)[(unsigned)index] = (unsigned char)ch;
	return ++index;
}

/*
 * Function parses the escape sequence pointed to by s and returns
 * the result. The type argument indicates whether the function is
 * called on a string or character constant. -1 is returned on
 * failure (bad format.)
 */
static int 
get_escape_sequence(const char *s, int type) {
	int	rc = -1;

	if (isdigit((unsigned char)*s)) {
		/* Octal constant */
		if (s[0] == '0' && isdigit((unsigned char)s[1]) == 0) {
			/* Null constant '\0' */
			return 0;
		}
		if (s[0] == '0') ++s;
		if (sscanf(s, type == TOK_STRING_LITERAL ? "%3o" : "%o",
					(unsigned *)&rc) != 1) {
			lexerror("Invalid octal constant.");
			return -1;
		}
	} else if (tolower((unsigned char)*s) == 'x') {
		/* Is hexadecimal constant */
		if (sscanf(s+1, type == TOK_STRING_LITERAL ? "%2x" : "%x",
					(unsigned *)&rc) != 1) {
			lexerror("Invalid hexadecimal constant.");
			return -1;
		}
	} else {
		/* Must be other escape sequence */
		static struct {
			char	ch;
			char	seq;
		} esc[] = {
			{ 'n', '\n' },
			{ 'r', '\r' },
			{ 't', '\t' },
			{ 'v', '\v' },
			{ 'b', '\b' },
			{ 'a', '\a' },
			{ 'f', '\f' },
			{ '\\', '\\' },
			{ '\'', '\''},
			{ '\"', '\"'},
			{ '\?', '\?'},
			{ 0, 0		}
		};
		int	i;
		for (i = 0; esc[i].ch != 0; ++i) {
			if (s[0] == esc[i].ch) {
				rc = esc[i].seq;
				if (s[1] != 0 && type == TOK_CHAR_LITERAL) {
					if (s[1] != '\'') {
						lexwarning("Multiple characters "
							"in character constant"
							" - ignoring excess"
							"characters.");
					}
				}
				break;
			}
		}
		if (esc[i].ch == 0) {
			lexerror("Invalid escape sequence.");
		}
	}

	if (type == TOK_CHAR_LITERAL
		&& (rc > UCHAR_MAX || rc < CHAR_MIN)) {
		lexwarning("Literal out of character range.");
	}
	rc = (char)rc;
	return rc;
}

/*
 * Reads trigraph from stream f, if present, parses it and returns the
 * result. This function must be called AFTER a ``?'' character has
 * been read from f. The function has three possible types of a return
 * value:
 * 1) -1 - This means a non-question-mark character was read. f remains
 *        the same as it was upon entry of the function
 * 2) 0 - This means a question mark character was read, but the
 *        next character did not complete a valid trigraph. f points to
 *        that non-question-mark character
 * 3) everything else - This is the character to which the trigraph maps
 *                      f points after the trigraph
 */
int 
get_trigraph(FILE *f) {
	static struct {
		char	sigch; /* significant character */
		char	trans; /* translation result of trigraph */
	} trig_map[] = {
		{ '=', '#' },
		{ ')', ']' },
		{ '!', '|' },
		{ '(', '[' },
		{ '\'', '^' },
		{ '>', '}' },
		{ '/', '\\' },
		{ '<', '{' },
		{ '-', '~' },
		{ 0, 0 }
	};
	int	ch;
	int	i;

	if ((ch = FGETC(f)) == EOF) {
		lexerror("Unexpected end of file");
		return 0;
	}
	if (ch != '?') {
		/* This has nothing to do with trigraphs at all... */
		UNGETC(ch, f);
		return -1;
	}

	/* If the next char matches, we have a trigraph */
	if ((ch = FGETC(f)) == EOF) {
		lexerror("Unexpected end of file");
		return 0;
	}
	for (i = 0; trig_map[i].sigch != 0; ++i) {
		if (trig_map[i].sigch == ch) {
			/* Is a trigraph! */
			return trig_map[i].trans;
		}
	}

	/* Not a trigraph */
	UNGETC(ch, f);
	return 0;
}

/*
 * Reads character literal from stream f, parses it and returns
 * the result. This function must be called AFTER the opening '
 * has already been read from f. -1 is returned on failure (bad
 * format / no closing ')
 */
int 
get_char_literal(FILE *f, int *err, char **text) {
	int	ch;
	int	trig;
	int	last = 0;
	int	rc;
	char	*p = NULL;

	*err = 0;
	/* Initialize store_char()'s private data */
	store_char(NULL, 0);
	while ((ch = FGETC(f)) != EOF) {
		/*
		 * Loop until closing ' encountered. It's important to check 
		 * that no \' escape sequence is considered to be a closing '.
		 *
		 * UPDATE: checking for last = \ did not suffice and hid a
		 * really obscure bug. If a literal reads '\\', then obviously
		 * last should not make the loop continue
		 */
		if (ch == '?') {
			/* Might be trigraph */
			if ((trig = get_trigraph(f)) == -1) {
				/*
				 * This is an illegal multi-char literal only if
				 * the current byte at f is not a ' char!
				 */
				store_char(&p, ch);
				last = ch;
			} else if (trig == 0) {
				/* Illegal, but let code later on catch it */
				store_char(&p, ch);
				store_char(&p, '?');
				last = '?';
			} else {
				/* Legal trigraph */
				store_char(&p, trig);
				last = trig;
			}
			continue;
		} else if (ch != '\'') {
			store_char(&p, ch);
			last = ch;
			continue;
		} else if (last == '\\') {
			if (!(p[0] == '\\' && p[1] == '\\')) {
				store_char(&p, ch);
				last = ch;
				continue;
			}
		}

		/* Literal ends here, parse and return it now */

		if (p == NULL) {
			lexerror("Empty character literal");
			*err = 1;
			return -1;
		}

		store_char(&p, 0);
		*text = p;

		if (p[0] == '\\') {
			/* Is escape sequence */
			rc = get_escape_sequence(p+1, TOK_CHAR_LITERAL); 
#ifdef DEBUG
			printf("Read character literal %d\n", rc);
#endif
			return rc;
		} else {
			/* Ordinary character */
			if (p[1] != 0) {
				/*
				 * Multi-character character literals are
				 * uncommon but legal in C (and illegal in
				 * C++.) Since their values are
				 * implementation-defined, it is OK to
				 * truncate them
				 */
				lexwarning("Multiple characters in character "
					"constant - ignoring excess "
					"characters.");
			}
#ifdef DEBUG
			printf("Read character literal ``%c''\n", p[0]);
#endif
			return p[0];
		}
	}
	lexerror("Unexpected end of file - expected ' to close "
		"character constant.");
	*err = 1;
	return -1;
}


/*
 * Reads string literal from stream f and returns a dynamically
 * allocated string as result. Must be called AFTER the opening
 * " has been read from f. A null pointer is returned on failure
 * (no closing ")
 */
char *
get_string_literal(FILE *f) {
	char	*p			= NULL;
	int	ch;
	int	trig;

	/* Initialize store_char()'s private data */
	store_char(NULL, 0);
	store_char(&p, '"');
	while ((ch = FGETC(f)) != EOF) {
		if (ch == '?') {
			/* Might be trigraph */
			if ((trig = get_trigraph(f)) == -1) {
				store_char(&p, ch);
				continue;
			} else if (trig == 0) {
				store_char(&p, ch);
				store_char(&p, '?');
				continue;
			} else {
				/* Is trigraph! Let code below handle it */
				ch = trig;
			}
		}

		if (ch == '"') {
			/* Literal ends here, nul-terminate and return it */
			store_char(&p, ch);
			store_char(&p, 0);
#ifdef DEBUG
			printf("Read string literal `%s'\n", p);
#endif
			return p;
		}
		store_char(&p, ch);
		if (ch == '\n' || ch == '\r') {
			lexerror("Newline in string literal - use '\\n' instead");
		} else if (ch == '\\') {
			/*
			 * Escape sequence - read next char to ensure " is
			 * escaped correctly
			 */

			char	buf[4];

#if 0
			/*
			 * Unfortunately, some escape sequence parsing must be
			 * done here already, because it wouldn't otherwise be
			 * known how many characters can be read (octal /
			 * hexadecimal constants take two, newlines, formfeeds,
			 * etc only one)
			 */
#endif
			if ((ch = FGETC(f)) == EOF) {
				break;
			}
			store_char(&p, ch);
#if 0
			buf[0] = (unsigned char)ch;
			if (tolower((unsigned char)buf[0]) != 'x'
				&& !isdigit(buf[0])) {	
				buf[1] = 0;
				ch = get_escape_sequence(buf,
					TOK_STRING_LITERAL);
			} else {
				/*
				 * Read numeric components of escape sequence.
				 * We must be careful to exclude non-digit
				 * characters from processing
				 */
				if ((ch = FGETC(f)) == EOF) {
					break;
				}
				if (is_part_of_constant(ch, buf[0])) {
					buf[1] = (unsigned char)ch;
					if ((ch = FGETC(f)) == EOF) {
						break;
					}
					if (is_part_of_constant(ch, buf[0])) {
						buf[2] = (unsigned char)ch;
						buf[3] = 0;
					} else {
						UNGETC(ch, f);
						buf[2] = 0;
					}
				} else {
					UNGETC(ch, f);
					if (buf[0] == 'x') {
						lexwarning("Empty hexadecimal "
							" escape sequence");
						continue;
					} else {
						buf[1] = 0;
					}	
				}
				ch = get_escape_sequence(buf,
						TOK_STRING_LITERAL);
			}
			if (ch == -1) {
				store_char(&p, '\x1');
			} else {
				store_char(&p, ch);
			}
#endif
			continue;
		}
#if 0
		store_char(&p, ch);
#endif
	}

	lexerror("Unexpected end of file - expected \" to close string literal.");
	if (p) free(p);
	return NULL;
}

/*
 * Returns the numeric code corresponding to ascii operator ``op''
 * on success, -1 if the operator is unknown
 */
static int 
do_get_operator(char *op, struct operator **opp) {
	int	i;
	int	c = *op;

	if ((i = LOOKUP_OP(c)) == 0) {
		return -1;
	}

	for (; operators[i].name[0] == c; ++i) {
		if (strcmp(operators[i].name, op) == 0) {
			*opp = &operators[i];
			if (operators[i].is_ambig != 0) {
				return operators[i].is_ambig;
			} else {
				return operators[i].value;
			}
		}
	}
		
	return -1;
}

/*
 * Read operator, starting from ch, from stream f. The result is
 * merely an approximation to the real semantics of the operator
 * in cases where multiple interpretations are possible in
 * different contexts (e.g. ``&'' as address-of operator vs. ``&''
 * as bitwise and.) The reported value for one of those ambiguous
 * operators must be adjusted at the hierarchial source analysis.
 * In particular, ambiguities occur with these operators:
 *
 * +	- unary plus    |  addition
 * -    - unary minus   |  substraction
 * *    - multiply      |  dereference       (pointer declaration)
 * &    - bitwise and   |  address-of
 * ++   - pre-increment |  post-increment
 * --   - pre-decrement |  post-decrement
 * :    - cond. oper.                        (bitfield member | label)
 */
int	
get_operator(int ch, FILE *f, struct operator **op) {
	char	buf[8];
	int	i = 1;
	int	tmp;
	int	latest_valid = -1;

	buf[0] = ch;
	buf[1] = 0;
	latest_valid = do_get_operator(buf, op);

	/*
	 * The strategy is to read new characters until the resulting
	 * string is NOT a valid operator anymore (or an identifier /
	 * whitespace is encountered.) The effect is that the previous
	 * operator (latest_valid) must have been valid, because we
	 * otherwise would have never reached this point. This is very
	 * handy for operators that, e.g., turn from ``<'' to ``<<''
	 * to ``<<=''.
	 */
	while ((ch = FGETC(f)) != EOF) {
		if (isalnum((unsigned char)ch)
			|| (ch != 0 && strchr("_$ \t\n", ch) != NULL)) {
			/* Identifier or whitespace reached */
			UNGETC(ch, f);
#ifdef DEBUG
			buf[i] = 0;
			printf("Read operator %s\n", buf);
#endif
			return latest_valid;
		}

		buf[i] = ch;
		buf[i + 1] = 0;
		if ((tmp = do_get_operator(buf, op)) == -1) {
			/* Not valid anymore - return last index */
			UNGETC(ch, f);
#ifdef DEBUG
			buf[i] = 0;
			printf("Read operator %s\n", buf);
#endif
			return latest_valid;
		}
		latest_valid = tmp;
		++i;
	}

	lexerror("Unexpected end of file.");
	return -1;
}


/*
 * Debugging function to print the value pointed to by ``ptr'',
 * interpreting it based on the type code ``type''. If the
 * ``verbose'' argument is nonzero, it will also print exactly
 * what type is being dealt with
 */
void 
rv_setrc_print(void *ptr, int type, int verbose) {
	(void)ptr; (void)type; (void)verbose;
#ifdef DEBUG
	if (type == TY_INT) { 
		printf("%s %d\n",
			verbose?"Read integer":"", *(int *)ptr);
	} else if (type == TY_UINT) {
		printf("%s %u\n",
			verbose?"Read unsigned integer":"", *(unsigned *)ptr);
	} else if (type == TY_LONG) {
		printf("%s %ld\n",
			verbose?"Read long":"", *(long *)ptr);
	} else if (type == TY_ULONG) {
		printf("%s %lu\n",
			verbose?"Read unsigned long":"", *(unsigned long *)ptr);
	} else if (type == TY_FLOAT) {
		printf("%s %f\n",
			verbose?"Read float":"", *(float *)ptr);
	} else if (type == TY_DOUBLE) {
		printf("%s %f\n",
			verbose?"Read double":"", *(double *)ptr);
	} else if (type == TY_LDOUBLE) {
		printf("%s %Lf\n",
			verbose?"Read long double":"", *(long double *)ptr);
	}
#endif
}



/* XXX nonsenseical now */
#define RV_ALLOC(ptr, type) \
do { 									\
	ptr = (type *)n_xmalloc(/*sizeof(type)*/16);	\
} while (0)

#define RV_SETRC(n, ptr, ty) \
do {								\
	n = n_xmalloc(sizeof *n);			\
	n->value = ptr;					\
	n->type  = ty;					\
	rv_setrc_print(ptr, ty, 1);		\
	return n;				 		\
} while (0)

	
static struct num * 
read_value(char *p, int octal_flag, int hexa_flag, int long_flag,
			int unsigned_flag, int float_flag, int fp_flag) {
	union {
		int			*iptr;
		unsigned int		*uiptr;
		long			*lptr;
		unsigned long		*ulptr;
		float			*fptr;
		double			*dptr;
		long double		*ldptr;

		/* using C99 types */
#if defined HAVE_LLONG || defined LLONG_MAX
#ifndef HAVE_LLONG
#define HAVE_LLONG
#endif
		long long		*llptr;
		unsigned long long	*ullptr;
#endif
	}			uptr;
	struct num	*rc;

	rc = n_xmalloc(sizeof *rc);

	if (hexa_flag == 1) {
		if (fp_flag == 1) {
			lexerror("Floating point in hexadecimal constant.");
			return NULL;
		}
		if (long_flag != 0) {
			int	ty;

			if (long_flag == 2) {
				ty = TY_LLONG;
			} else {
				ty = TY_LONG;
			}
			if (long_flag != 2) {
				if (unsigned_flag == 1) {
					ty = TY_ULONG;
					RV_ALLOC(uptr.ulptr, unsigned long);
					if (sscanf(p, "%lx", uptr.ulptr) != 1) {
						lexerror("Bad unsigned long "
						"hexadecimal constant.");
						free(uptr.ulptr);
						return NULL;
					}
					RV_SETRC(rc, uptr.ulptr, ty);
				} else {
					RV_ALLOC(uptr.lptr, long);
					if (sscanf(p, "%lx",
					(unsigned long *)uptr.lptr) != 1) {
						lexerror("Bad long hexadecimal "
							"constant.");
						free(uptr.lptr);
						return NULL;
					}
					RV_SETRC(rc, uptr.lptr, ty);
				}
			} else {
#ifdef HAVE_LLONG
				ty = TY_LLONG;
				if (unsigned_flag == 1) {
					ty = TY_ULLONG;
					RV_ALLOC(uptr.ullptr,
						unsigned long long);
					if (sscanf(p, "%llx", uptr.ullptr)
						!= 1) {
						lexerror("Bad unsigned long long"
							" hex constant.");
						free(uptr.ullptr);
						return NULL;
					}
					RV_SETRC(rc, uptr.ullptr, ty);
				} else {
					RV_ALLOC(uptr.llptr, long long);
					if (sscanf(p, "%llx",
				(unsigned long long *)uptr.llptr) != 1) {
						lexerror("Bad long long "
						"hexadecimal constant.");
						free(uptr.llptr);
						return NULL;
					}
					RV_SETRC(rc, uptr.llptr, ty);
				}
#else
				lexerror("This compiler wasn't built with " 
						"long long support");
#endif
			}
		} else {
			if (unsigned_flag == 1) {
				RV_ALLOC(uptr.uiptr, unsigned int);
				if (sscanf(p, "%x", uptr.uiptr) != 1) {
					lexerror("Bad hexadecimal constant.");
					free(uptr.uiptr);
					return NULL;
				}
				RV_SETRC(rc, uptr.uiptr, TY_UINT);
			} else {
				RV_ALLOC(uptr.iptr, int);
				if (sscanf(p, "%x", (unsigned *)uptr.iptr)
					!= 1) {
					lexerror("Bad hexadecimal constant.");
					free(uptr.iptr);
					return NULL;
				}
				RV_SETRC(rc, uptr.iptr, TY_INT);
			}
		}
	} else if (octal_flag == 1) {
		if (fp_flag == 1) {
			lexerror("Floating point in octal constant.");
			return NULL;
		}

		if (long_flag != 0) {
			int	ty;

			if (long_flag == 2) {
				ty = TY_LLONG;
			} else {
				ty = TY_LONG;
			}
			if (long_flag != 2) {
				if (unsigned_flag == 1) {
					ty = TY_ULONG;
					RV_ALLOC(uptr.ulptr, unsigned long);
					if (sscanf(p, "%lo", uptr.ulptr) != 1) {
						free(uptr.ulptr);
						lexerror("Bad unsigned long "
							"octal constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.ulptr, ty);
				} else {
					RV_ALLOC(uptr.lptr, long);
					if (sscanf(p, "%lo",
					(unsigned long *)uptr.lptr) != 1) {
						free(uptr.lptr);
						lexerror("Bad long octal "
							"constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.lptr, ty);
				}
			} else {
#ifdef HAVE_LLONG
				if (unsigned_flag == 1) {
					ty = TY_ULLONG;
					RV_ALLOC(uptr.ullptr,
						unsigned long long);
					if (sscanf(p, "%llo", uptr.ullptr)
						!= 1) {
						free(uptr.ullptr);
						lexerror("Bad unsigned long long"
							" octal constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.ullptr, ty);
				} else {
					RV_ALLOC(uptr.llptr, long long);
					if (sscanf(p, "%llo", uptr.llptr)
						!= 1) {
						free(uptr.llptr);
						lexerror("Bad long long octal "
							"constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.llptr, ty);
				}
#else
				lexerror("This compiler wasn't built with " 
						"long long support");
#endif
			}
		} else {
			if (unsigned_flag == 1) {
				RV_ALLOC(uptr.uiptr, unsigned int);
				if (sscanf(p, "%o", uptr.uiptr) != 1) {
					free(uptr.uiptr);
					lexerror("Bad octal constant.");
					return NULL;
				}
				RV_SETRC(rc, uptr.uiptr, TY_UINT);
			} else {
				RV_ALLOC(uptr.iptr, int);
				if (sscanf(p, "%o", (unsigned *)uptr.iptr)
					!= 1) {
					free(uptr.iptr);
					lexerror("Bad octal constant.");
					return NULL;
				}
				RV_SETRC(rc, uptr.iptr, TY_INT);
			}
		}	
	} else if (fp_flag) {
		if (float_flag) {
			RV_ALLOC(uptr.fptr, float);
			if (sscanf(p, "%f", uptr.fptr) != 1) {
				free(uptr.fptr);
				lexerror("Bad floating point constant.");
				return NULL;
			}
			RV_SETRC(rc, uptr.fptr, TY_FLOAT);
		} else if (long_flag == 1) {
			RV_ALLOC(uptr.ldptr, long double);
			if (sscanf(p, "%Lf", uptr.ldptr) != 1) {
				free(uptr.ldptr);
				lexerror("Bad long double constant.");
				return NULL;
			}
			RV_SETRC(rc, uptr.ldptr, TY_LDOUBLE);
		} else {
			RV_ALLOC(uptr.dptr, double);
			if (sscanf(p, "%lf", uptr.dptr) != 1) {
				free(uptr.dptr);
				lexerror("Bad double constant.");
				return NULL;
			}
			RV_SETRC(rc, uptr.dptr, TY_DOUBLE);
		}
	} else {
		/* Must be decimal */
		if (long_flag != 0) {
			int	ty;

			if (long_flag == 2) {
				ty = TY_LLONG;
			} else {
				ty = TY_LONG;
			}
			if (long_flag != 2) {
				if (unsigned_flag == 1) {
					ty = TY_ULONG;
					RV_ALLOC(uptr.ulptr, unsigned long);
					if (sscanf(p, "%lu", uptr.ulptr) != 1) {
						free(uptr.ulptr);
						lexerror("Bad unsigned long "
							"constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.ulptr, ty);
				} else {
					RV_ALLOC(uptr.lptr, long);
					if (sscanf(p, "%ld", uptr.lptr) != 1) {
						free(uptr.lptr);
						lexerror("Bad long constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.lptr, ty);
				}
			} else {
#ifdef HAVE_LLONG
				if (unsigned_flag == 1) {
					ty = TY_ULLONG;
					RV_ALLOC(uptr.ullptr,
						unsigned long long);
					if (sscanf(p, "%llu", uptr.ullptr)
						!= 1) {
						free(uptr.ullptr);
						lexerror("Bad unsigned long "
							"long constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.ullptr, ty);
				} else {
					RV_ALLOC(uptr.llptr, long long);
					if (sscanf(p, "%lld", uptr.llptr)
						!= 1) {
						free(uptr.llptr);
						lexerror("Bad long long "
							"constant.");
						return NULL;
					}
					RV_SETRC(rc, uptr.llptr, ty);
				}
#else
				lexerror("This compiler wasn't built with " 
						"long long support");
#endif
			}
		} else {
			if (unsigned_flag == 1) {
				RV_ALLOC(uptr.uiptr, unsigned int);
				if (sscanf(p, "%u", uptr.uiptr) != 1) {
					free(uptr.uiptr);
					lexerror("Bad unsigned decimal constant.");
					return NULL;
				}
				RV_SETRC(rc, uptr.uiptr, TY_UINT);
			} else {
				RV_ALLOC(uptr.iptr, int);
				if (sscanf(p, "%d", uptr.iptr) != 1) {
					free(uptr.iptr);
					lexerror("Bad decimal constant.");
					return NULL;
				}
				RV_SETRC(rc, uptr.iptr, TY_INT);
			}
		}
	}
	return NULL;
}

#define STORE_CHAR(p, ch) do { \
		store_char(p, ch); \
} while (0)
			
	

/*
 * Reads a numeric literal from stream f and returns a pointer to
 * a dynamically allocated ``struct num'' containing its value and
 * type. The first character that has already been read from the
 * literal (and which actually indicated that we are dealing with
 * one) is passed as ``firstch''. On failure (bad format or memory
 * allocation problems) a null pointer is returned. 
 * This stuff is believed to be able to handle all types of numeric
 * constants that exist in C
 */ 
struct num *
get_num_literal(int firstch, FILE *f) {
	int			ch;
	int			real_type;
	int			octal_flag		= 0;
	int			hexa_flag		= 0;
	int			long_flag		= 0;
	int			unsigned_flag		= 0;
	int			float_flag		= 0;
	int			fp_flag			= 0;
	int			digits_read		= 0;
	int			last_dig		= 0;
	char			*p			= NULL;
	struct num		*rc;

	/* Initialize store_char() */
	store_char(NULL, 0);

	store_char(&p, firstch);
	if (firstch == '0') {
		if ((ch = FGETC(f)) != EOF) {
			if (tolower((unsigned char)ch) == 'x') {
				/* hexadecimal (0x1) */
				hexa_flag = 1;
				store_char(&p, ch);
			} else if (ch == '.') {
				/* floating point (0.1) */
				fp_flag = 1;
				store_char(&p, ch);
			} else if (isdigit((unsigned char)ch)) {
				/* octal (01) */
				octal_flag = 1;
				store_char(&p, ch);
				if (ch != '0') { /* 0 is insignificant */
					++digits_read;
				} else {
					last_dig = ch;
				}
			} else {
				/* unknown */
				UNGETC(ch, f);
				++digits_read;
			}
		}
	} else if (firstch == '.') {
		/* Number like .5 (equivalent to 0.5) */
		STORE_CHAR(&p, ch);
		fp_flag = 1;
	} else {
		/* Is significant digit */
		++digits_read;
		last_dig = firstch;
	}


	while ((ch = FGETC(f)) != EOF) {
		switch (ch) {
		case 'l':
		case 'L':
			++long_flag;
			STORE_CHAR(&p, ch);
			if (long_flag > 2) {
				lexerror("Unknown constant (was expecting "
					"long long)");
				return NULL;
			}
			break;
		case 'u':
		case 'U':
			++unsigned_flag;
			STORE_CHAR(&p, ch);
			if (unsigned_flag > 1) {
				lexwarning("Multiple unsigned designators "
					"in constant");
			}
			break;
		case 'f':
		case 'F':
			if (hexa_flag) {
				store_char(&p, ch);
				++digits_read;
			} else {
				float_flag = ++fp_flag;
				STORE_CHAR(&p, ch);
				if (float_flag > 2) {
					lexwarning(
						"Multiple floating point "
						"designators in constant"); 
				}
			}
			break;
		case '.':
			fp_flag = 1;
			store_char(&p, '.');
			break;
		default:
			if (isdigit((unsigned char)ch)) {
				/*
				 * Verify the numeric part has not yet been
				 * termianted
				 */
				if (long_flag || unsigned_flag || float_flag) {
					lexerror("Unexpected digit.");
					if (p) free(p);
					return NULL;
				}
				store_char(&p, ch);
				++digits_read;
				if (last_dig == 0 && ch != '0') {
					last_dig = ch;
				}
				continue;
			}

			/* Not digit - Does the constant end here? */
			if (tolower((unsigned char)ch) == 'e') {
				if (hexa_flag == 0) {
					/*
					 * Must be scientific notation
					 */
					if (fp_flag == 0) {
						lexerror("Using scientific "
							"notation on "
							"integral value.");
						if (p) free(p);
						return NULL;
					}
					store_char(&p, ch);
					if ((ch = FGETC(f)) == '+'
						|| ch == '-') {
						store_char(&p, ch);
					} else {
						UNGETC(ch, f);
					}
					continue;
				} else {
					++digits_read;
				}
				store_char(&p, ch);
				continue;
			}

			/* Hexadecimal digit? */
			if (hexa_flag == 1) {
				int	t;

				t = tolower((unsigned char)ch);
				if (strchr("abcd", t) != NULL) {
					/* e & f are covered by other cases */
					store_char(&p, t);
					++digits_read;
					continue;
					/*break; ??? */
				}
			}

			/* Ok, done with it */

			store_char(&p, 0);
			UNGETC(ch, f);

			if (digits_read == 0 && !octal_flag) {
				lexerror("Constant with no digits");
			} else if (fp_flag == 0) {
				/* 
				 * Perform range check, adjust type of
				 * constant as needed
				 */
				real_type = range_check(p, hexa_flag,
					octal_flag, unsigned_flag, long_flag,
					digits_read);
				if (real_type == -1) {
					free(p);
					return NULL;
				} else if (real_type != 0)  {
					/* 0 means unchanged */
					if (real_type == TY_ULONG
						|| real_type == TY_ULLONG
						|| real_type == TY_UINT) {
						unsigned_flag = 1;
					}
					if (real_type == TY_LONG
						|| real_type == TY_ULONG) {
						long_flag = 1;
					} else if (real_type == TY_LLONG
						|| real_type == TY_ULLONG) {
						long_flag = 2;
					}
				}
			}

			rc = read_value(p, octal_flag, hexa_flag, long_flag,
					unsigned_flag, float_flag, fp_flag);
#ifdef DEBUG
			printf("(%d digits)\n", digits_read);
#endif
			if (fp_flag /*&& backend->need_floatconst*/) {
#if 0
				struct ty_float		*fc;
				static unsigned long	count;

				fc = n_xmalloc(sizeof *fc);
				fc->count = count++;
				fc->num = rc;
				fc->next = float_const;
				float_const = fc;
#endif
			}	
			rc->ascii = p;
					
			return rc;
		}
	}

	lexerror("Unexpected end of file - expected end of numeric constant.");
	if (p) free(p);
	return NULL;
}

/*
 * Reads an identifier from stream f and returns a pointer to a 
 * dynamically allocated string containing the result. It will
 * append all characters until a character is encountered that
 * is neither alpha-numeric, nor ``_'', nor ``$''. On failure
 * (end of file), a null pointer is returned
 */
char *
get_identifier(int ch, FILE *f) {
	char	*p = NULL;

	/* Initialize static store_char() data */
	store_char(NULL, 0);

	store_char(&p, ch);
	while ((ch = FGETC(f)) != EOF) {
		if (isalnum((unsigned char)ch) || ch == '_' || ch == '$') {
			store_char(&p, ch);
			if (ch == '$') {
				lexerror("`$' characters are not allowed in "
					"identifiers");
			}
		} else {
			/*
			 * Operator, white space or other token encountered -
			 * return
			 */
			store_char(&p, 0);
			UNGETC(ch, f);
#ifdef DEBUG
			printf("Read identifier ``%s''\n", p);
#endif
			return p;
		}
	}

	lexerror("Unexpected end of file - expected termination of identifier.");
	return NULL;
}


static char 	*curfile;
static int	curfileid;

/*
 * Set path of file currently processed. This will be used by
 * store_token(), which saves in each token which file it is
 * contained by
 * XXX This is sorta bogus??
 */
void
token_setfile(char *file) {
	curfile = file;
}

void
token_setfileid(int id) {
	curfileid = id;
}	


static char *
tok_to_ascii(int type, void *data) {
	char	*ret = "???";

	if (type == TOK_IDENTIFIER) {
		if (data != NULL) ret = data;
		else ret = "identifier";
	} else if (type == TOK_STRING_LITERAL) {
#if 0 
		if (data != NULL) {
			struct ty_string	*str;
			str = data;
			ret = str->str;
		} else {
			ret = "string constant";
		}
#endif
		ret = data;
	} else if (type == TOK_OPERATOR) {
		if (data != NULL) {
			ret = lookup_operator(*(int *)data);
			if (ret == NULL) {
				(void) fprintf(stderr,
					"FATAL - unknown operator - %d\n",
					*(int *)data);
				abort();
			}
		} else {
			ret = "operator";
		}
	} else if (type == TOK_PAREN_OPEN) {
		ret = "(";
	} else if (type == TOK_PAREN_CLOSE) {
		ret = ")";
	} else if (type == TOK_COMP_OPEN) {
		ret = "{";
	} else if (type == TOK_COMP_CLOSE) {
		ret = "}";
	} else if (type == TOK_ARRAY_OPEN) {
		ret = "[";
	} else if (type == TOK_ARRAY_CLOSE) {
		ret = "]";
	} else if (type == TOK_SEMICOLON) {
		ret = ";";
	} else if (type == TOK_WS) {
		ret = data;
	} else if (IS_CONSTANT(type)) {
		ret = "(constant)";
	} else if (IS_OPERATOR(type)) {
		/* This will only be used by expect_token() */
	} else {
#ifdef DEBUG
		char	lame[256];
		sprintf(lame, "<unknown>(code = %d)", type);
		ret = strdup(lame);
#else
		ret = "???";
#endif
	}
	return ret;
}


/*
 * Appends the token specified by the data-type-linenum triple to
 * the token list pointed to by ``dest''. Exits with an error
 * message if no memory can be allocated. ``data'' MUST point to
 * dynamically allocated memory if you intend use destroy_toklist()
 * token_setfile() must be called to validate the file state data
 * before store_token() can be used
 * Adjacent string literals are concatenated
 */
struct token *
store_token(struct token **dest, struct token **dest_tail,
	void *data, int type, int linenum) {

	struct token		*t;

	if (*dest_tail == (void *)0x40) abort();
	if (*dest == NULL) {
		*dest = alloc_token();
		t = *dest;
		*dest_tail = t;
		t->prev = NULL;
		t->next = NULL;
	} else {
		(*dest_tail)->next = alloc_token();
		(*dest_tail)->next->prev = *dest_tail;
		t = (*dest_tail)->next;
		t->next = NULL;
		*dest_tail = t;
	}

	/* Save token in ascii */
	t->ascii = tok_to_ascii(type, data);

	t->file = curfile; 
	t->fileid = curfileid;
#if 0
	if (data == NULL) {
		/* We're done - terminate list */
		t->type = 0;
		t->data = NULL;
		t->line = linenum;
		return t;
	}
#endif
	t->type = type;
	t->line = linenum;

	if (float_const != NULL &&
		(type == TY_FLOAT
		|| type == TY_DOUBLE
		|| type == TY_LDOUBLE)) {
		/* XXX ugly way to pass the data :( */
		t->data = float_const;
	} else {	
		t->data = data;
	}	

	t->line_ptr = lex_line_ptr;
	t->tok_ptr = lex_tok_ptr;

	if (type == TOK_IDENTIFIER) {
		/* Let's try and see whether this identifier is a keyword */
	}
	if (t->type == 0) abort();
	return t;
}


/*
 * Advances token list pointed to by tok to next node if possible.
 * Returns 1 if the end of the list was reached unexpectedly, else 0
 */
int 
next_token(struct token **tok) {
	struct token	*t = *tok;

	if (t == NULL || t->next == NULL) {
#if 0
		errorfl(t, "Unexpected end of file!");
#endif
		return 1;
	}
	*tok = t->next;
	return 0;
}

void
append_token_copy(struct token **d, struct token **d_tail, struct token *t) {
	t = n_xmemdup(t, sizeof *t);
	if (*d != NULL) {
		(*d_tail)->next = t;
		t->prev = *d_tail;
		*d_tail = t;
	} else {
		*d = *d_tail = t;
		t->prev = NULL;
	}	

	t->next = NULL;
}	

void
free_token_list(struct token *t) {
	return;
	while (t != NULL) {
		/* XXX free data members */
		struct token	*tmp = t->next;
		free(t);
		t = tmp;
	}
}

void
output_token_list(FILE *out, struct token *t) {
	for (; t != NULL; t = t->next) {
		x_fprintf(out, "%s", t->ascii);
	}
}

char *
toklist_to_string(struct token *toklist) {
        struct token    *t;
        char            *p = NULL;
        size_t          len = 0;
        size_t          tlen;

        for (t = toklist; t != NULL; t = t->next) {
                tlen = strlen(t->ascii);
                p = n_xrealloc(p, len+tlen+1);
                sprintf(p+len, "%s", t->ascii);
                len += tlen;
        }
        return p;
}