FWD » Core Development » Conversion Tools

lexer grammar ProgressLexer;

@header{
import java.util.*;
}

@members {
public static Map<Integer, String> types = new HashMap<>();
public static List<String> keywords = new ArrayList<>();
static
{
    types.put(MESSAGE, "keyword");
    types.put(DEFINE, "keyword");
    types.put(TEXT, "variable");
    types.put(LCURLY, "delimiter");
    types.put(AMP, "delimiter");
    types.put(RCURLY, "delimiter");
    types.put(DQUOTE, "delimiter");
    types.put(DOT, "operator");
    types.put(STRING_LITERAL, "string");
}
static
{
    keywords.add("message");
    keywords.add("scoped-define");
    keywords.add("global-define");
}
}
MESSAGE : 'message' | 'MESSAGE' ;
DEFINE : 'scoped-define' | 'global-define' ;
TEXT : (~[ {}&"\r\n])+ ;
LCURLY : '{' ;
AMP : '&' ;
RCURLY : '}' ;
DQUOTE : '"' ;
DOT : '.' ;
STRING_LITERAL : DQUOTE (TEXT | ' ')* DQUOTE ;
WS : [ \t\r\n]+;

import org.eclipse.lsp4j.*;
import org.eclipse.lsp4j.jsonrpc.messages.Either;
import org.eclipse.lsp4j.services.LanguageClient;
import org.eclipse.lsp4j.services.LanguageClientAware;
import org.eclipse.lsp4j.services.LanguageServer;
import org.eclipse.lsp4j.services.TextDocumentService;
import org.eclipse.lsp4j.services.WorkspaceService;

import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;

import static grammar.AntlrLspAdapter.*;

/**
 * Language Server implementation for Ballerina.
 */
public class ProgressLanguageServer implements LanguageServer, LanguageClientAware {

    private TextDocumentService textDocumentService;
    private WorkspaceService workspaceService;
    private ClientCapabilities clientCapabilities;
    LanguageClient languageClient;
    private int shutdown = 1;

    public ProgressLanguageServer() {
        this.textDocumentService = new ProgressTextDocumentService(this);
        this.workspaceService = new ProgressWorkSpaceService(this);
    }

   private void setupSemanticTokensCapabilities(ServerCapabilities capabilities) {
      SemanticTokensWithRegistrationOptions semanticTokensCapabilities = new SemanticTokensWithRegistrationOptions();

      // set up the legend (token types and modifiers)
      SemanticTokensLegend legend = new SemanticTokensLegend(TOKEN_TYPES, TOKEN_MODIFIERS);

      semanticTokensCapabilities.setLegend(legend);

      // enable full document semantic tokens
      semanticTokensCapabilities.setFull(Either.forLeft(true));

      // enable range semantic tokens
      semanticTokensCapabilities.setRange(Either.forLeft(true));

      capabilities.setSemanticTokensProvider(semanticTokensCapabilities);
   }

    @Override
    public CompletableFuture<InitializeResult> initialize(InitializeParams initializeParams) {

        final InitializeResult response = new InitializeResult(new ServerCapabilities());
        //Set the document synchronization capabilities to full.
        response.getCapabilities().setTextDocumentSync(TextDocumentSyncKind.Full);
        this.clientCapabilities = initializeParams.getCapabilities();

        /* check if dynamic registration of completion capability is allowed by the client. If so we don't register the capability.
           else, we register the completion capability.
         */
        if (!isDynamicCompletionRegistration()) {
            response.getCapabilities().setCompletionProvider(new CompletionOptions());
        }

        setupSemanticTokensCapabilities(response.getCapabilities());
        response.getCapabilities().setDefinitionProvider(true);
        return CompletableFuture.supplyAsync(() -> response);
    }

    @Override
    public void initialized(InitializedParams params) {
        // check if dynamic completion support is allowed, if so register.
        if (isDynamicCompletionRegistration()) {
            CompletionRegistrationOptions completionRegistrationOptions = new CompletionRegistrationOptions();
            Registration completionRegistration = new Registration(UUID.randomUUID().toString(),
                    "textDocument/completion", completionRegistrationOptions);
            languageClient.registerCapability(new RegistrationParams(List.of(completionRegistration)));
        }
    }

    @Override
    public CompletableFuture<Object> shutdown() {
        shutdown = 0;
        return CompletableFuture.supplyAsync(Object::new);
    }

    @Override
    public void exit() {
        System.exit(shutdown);
    }

    @Override
    public TextDocumentService getTextDocumentService() {
        return this.textDocumentService;
    }

    @Override
    public WorkspaceService getWorkspaceService() {
        return this.workspaceService;
    }

    @Override
    public void connect(LanguageClient languageClient) {
        this.languageClient = languageClient;
        LSClientLogger.getInstance().initialize(this.languageClient);
    }

    private boolean isDynamicCompletionRegistration() {
        TextDocumentClientCapabilities textDocumentCapabilities =
                clientCapabilities.getTextDocument();
        return textDocumentCapabilities != null && textDocumentCapabilities.getCompletion() != null
                && Boolean.FALSE.equals(textDocumentCapabilities.getCompletion().getDynamicRegistration());
    }
}

import grammar.AntlrLspAdapter;
import grammar.SemanticToken;
import org.eclipse.lsp4j.*;
import org.eclipse.lsp4j.jsonrpc.messages.Either;
import org.eclipse.lsp4j.services.TextDocumentService;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;

/**
 * TextDocumentService implementation for Ballerina.
 */
public class ProgressTextDocumentService implements TextDocumentService {

    private ProgressLanguageServer languageServer;
    private LSClientLogger clientLogger;
    private AntlrLspAdapter antlrLspAdapter = new AntlrLspAdapter();
   private final Map<String, String> documentContents = new HashMap<>();

    public ProgressTextDocumentService(ProgressLanguageServer languageServer) {
        this.languageServer = languageServer;
        this.clientLogger = LSClientLogger.getInstance();
    }

    @Override
    public void didOpen(DidOpenTextDocumentParams params) {
       String uri = params.getTextDocument().getUri();
       String content = params.getTextDocument().getText();
       documentContents.put(uri, content);

       LSClientLogger.getInstance().logMessage("Document opened: " + uri, MessageType.Info);
    }

    @Override
    public void didChange(DidChangeTextDocumentParams params) {
       String uri = params.getTextDocument().getUri();
       List<TextDocumentContentChangeEvent> changes = params.getContentChanges();

       // For full sync, we replace the entire content
       if (!changes.isEmpty()) {
          String newContent = changes.get(0).getText();
          documentContents.put(uri, newContent);
       }
    }

    @Override
    public void didClose(DidCloseTextDocumentParams params) {
       String uri = params.getTextDocument().getUri();
       documentContents.remove(uri);
        this.clientLogger.logMessage("Operation '" + "text/didClose" +
                "' {fileUri: '" + params.getTextDocument().getUri() + "'} Closed", MessageType.Error);
    }

    @Override
    public void didSave(DidSaveTextDocumentParams didSaveTextDocumentParams) {
        this.clientLogger.logMessage("Operation '" + "text/didSave" +
                "' {fileUri: '" + didSaveTextDocumentParams.getTextDocument().getUri() + "'} Saved", MessageType.Error);
    }

    @Override
    public CompletableFuture<Either<List<CompletionItem>, CompletionList>> completion(CompletionParams position) {
        var tokens = documentContents.get(position.getTextDocument().getUri())
                                     .split("\\s+");
        String lastToken = tokens[tokens.length - 1];

        return CompletableFuture.supplyAsync(() -> {
               clientLogger.logMessage(lastToken, MessageType.Info);

            return Either.forLeft(antlrLspAdapter.getCompletions(lastToken));
        });
    }

   /**
    * Provide semantic tokens for the full document using ANTLR lexer
    */
   @Override
   public CompletableFuture<SemanticTokens> semanticTokensFull(SemanticTokensParams params) {
      String uri = params.getTextDocument().getUri();
      String content = documentContents.get(uri);
      this.clientLogger.logMessage("LEXING!", MessageType.Error);
      if (content == null) {
         return CompletableFuture.completedFuture(new SemanticTokens(List.of()));
      }

      return CompletableFuture.supplyAsync(() -> {
         try {
            // Use ANTLR lexer to analyze the document content
            List<SemanticToken> tokens =
                    antlrLspAdapter.analyzeText(content);

            // Convert to LSP format
            List<Integer> data = antlrLspAdapter.tokensToLSPFormat(tokens);

            return new SemanticTokens(data);
         } catch (Exception e) {
            LSClientLogger.getInstance().logMessage(
                    "Error analyzing semantic tokens with ANTLR: " + e.getMessage(), MessageType.Error);
            return new SemanticTokens(List.of());
         }
      });
   }

   /**
    * Provide semantic tokens for the specified range.
    */
   @Override
   public CompletableFuture<SemanticTokens> semanticTokensRange(SemanticTokensRangeParams params)
   {
      String uri = params.getTextDocument().getUri();
      Range range = params.getRange();

      return CompletableFuture.supplyAsync(() -> {
         List<SemanticToken> tokens = antlrLspAdapter.analyzeTextRange(documentContents.get(uri), range);
         List<Integer> data = antlrLspAdapter.tokensToLSPFormat(tokens);
         return new SemanticTokens(data);
      });
   }

   /**
    * This is a mock method which always points to the first character of the file when executing
    * 'goto definition'
    * @param params DefinitionParams params
    * @return
    */
   @Override
   public CompletableFuture<Either<List<? extends Location>, List<? extends LocationLink>>> definition(DefinitionParams params) {
      String uri = params.getTextDocument().getUri();
      Range range = new Range(new Position(0, 0), new Position(0, 1));
      Location location = new Location(uri, range);

      return CompletableFuture.completedFuture(Either.forLeft(List.of(location)));
   }
}

public class AntlrLspAdapter {
   public final Map<Integer, Integer> ANTLR_TO_SEMANTIC_TYPE = new HashMap<>();

   // semantic token types mapping to ANTLR token types
   public static final List<String> TOKEN_TYPES = List.of(
           "keyword",      // MESSAGE, DEFINE
           "string",       // STRING_LITERAL
           "operator",     // AMP, DOT
           "delimiter",    // LCURLY, RCURLY, DQUOTE
           "variable",     // TEXT
           "comment"       // For future use
   );

   // semantic token modifiers
   public static final List<String> TOKEN_MODIFIERS = List.of(
           "declaration",
           "definition",
           "readonly",
           "static",
           "deprecated",
           "abstract",
           "async",
           "modification",
           "documentation",
           "defaultLibrary" 
   );

   /**
    * Analyze text using ANTLR lexer and return semantic tokens
    */
   public List<SemanticToken> analyzeText(String text) {
      List<SemanticToken> tokens = new ArrayList<>();

      try {
         // create ANTLR input stream
         ANTLRInputStream input = new ANTLRInputStream(text);

         // create lexer
         ProgressLexer lexer = new ProgressLexer(input);

         // remove default error listeners to avoid console output
         lexer.removeErrorListeners();

         // get all tokens
         CommonTokenStream tokenStream = new CommonTokenStream(lexer);
         tokenStream.fill();
         for (var e : lexer.types.entrySet()) {
            LSClientLogger.getInstance().logMessage(e.getKey() + "   " + e.getValue(), MessageType.Info);
         }
         ANTLR_TO_SEMANTIC_TYPE.putAll(lexer.types.entrySet().stream().collect(
                 Collectors.toMap(Map.Entry::getKey,
                         entry -> TOKEN_TYPES.indexOf(entry.getValue()))
         ));

         List<Token> antlrTokens = tokenStream.getTokens();

         for (Token token : antlrTokens) {
            // skip EOF token and whitespace
            if (token.getType() == Token.EOF || token.getType() == ProgressLexer.WS) {
               continue;
            }

            // convert ANTLR position to LSP position (0-based)
            int line = token.getLine() - 1; // ANTLR is 1-based, LSP is 0-based
            int startChar = token.getCharPositionInLine();
            int length = token.getStopIndex() - token.getStartIndex() + 1;

            // map ANTLR token type to semantic token type
            Integer semanticTokenType = ANTLR_TO_SEMANTIC_TYPE.get(token.getType());
            if (semanticTokenType == null) {
               // default to variable for unknown tokens
               semanticTokenType = TOKEN_TYPES.indexOf("variable");
            }
            int stt = semanticTokenType;
            // determine modifiers based on context (you can enhance this)

            int modifiers = getTokenModifiers(token);

            tokens.add(new SemanticToken() {
               @Override
               public int line() {
                  return line;
               }

               @Override
               public int startChar() {
                  return startChar;
               }

               @Override
               public int length() {
                  return length;
               }

               @Override
               public int tokenType() {
                  return stt;
               }

               @Override
               public int tokenModifiers() {
                  return modifiers;
               }
            });
         }

      } catch (Exception e) {
         // log error but don't fail completely
         System.err.println("Error analyzing tokens with ANTLR: " + e.getMessage());
      }

      return tokens;
   }

   /**
    * Determine token modifiers based on context
    */
   private int getTokenModifiers(Token token) {
      int modifiers = 0;

      if (token.getType() == ProgressLexer.DEFINE) {

         modifiers |= (1 << TOKEN_MODIFIERS.indexOf("definition"));
      }

      return modifiers;
   }

   /**
    * Convert semantic tokens to LSP format with delta encoding
    */
   public List<Integer> tokensToLSPFormat(List<SemanticToken> tokens) {
      List<Integer> data = new ArrayList<>();

      int prevLine = 0;
      int prevStart = 0;

      for (SemanticToken token : tokens) {
         // calculate deltas for LSP format
         int deltaLine = token.line() - prevLine;
         int deltaStart = deltaLine == 0 ? token.startChar() - prevStart : token.startChar();

         // LSP semantic tokens format: deltaLine, deltaStart, length, tokenType, tokenModifiers
         data.add(deltaLine);
         data.add(deltaStart);
         data.add(token.length());
         data.add(token.tokenType());
         data.add(token.tokenModifiers());

         prevLine = token.line();
         prevStart = token.startChar();
      }

      return data;
   }

   /**
    * Analyze text within a specific range
    */
   public List<SemanticToken> analyzeTextRange(String text, Range range) {
      // first get all tokens
      List<SemanticToken> allTokens = analyzeText(text);

      // filter tokens within the range
      List<SemanticToken> rangeTokens = new ArrayList<>();

      for (SemanticToken token : allTokens) {
         Position tokenStart = new Position(token.line(), token.startChar());
         Position tokenEnd = new Position(token.line(), token.startChar() + token.length());

         // check if token overlaps with the range
         if (isPositionInRange(tokenStart, range) || isPositionInRange(tokenEnd, range)) {
            rangeTokens.add(token);
         }
      }

      return rangeTokens;
   }

   private boolean isPositionInRange(Position pos, Range range) {
      Position start = range.getStart();
      Position end = range.getEnd();

      if (pos.getLine() < start.getLine() || pos.getLine() > end.getLine()) {
         return false;
      }

      if (pos.getLine() == start.getLine() && pos.getCharacter() < start.getCharacter()) {
         return false;
      }

      if (pos.getLine() == end.getLine() && pos.getCharacter() > end.getCharacter()) {
         return false;
      }

      return true;
   }

   public List<CompletionItem> getCompletions(String text) {
      List<CompletionItem> res = new ArrayList<>();
      for (var kw : ProgressLexer.keywords) {
         if (kw.startsWith(text)) {
            var completion = new CompletionItem();
            completion.setLabel(kw);
            completion.setInsertText(kw);
            res.add(completion);
         }
      }

      return res;
   }
}