Java How to get HTML code from URL including AJAX generated code

Issue

I'm trying to parse tables using Java, but I get HTML text without tables with data. Tell me the library or the right way to do it. Thank you for any help. Java-8

https://www.tcgplayer.com/categories/trading-and-collectible-card-games/magic-the-gathering/price-guides/khans-of-tarkir

I tried to find answers, but without success

Thank you! Java

Solution

You can just use their internal api

call this get api, and supply the amount of rows (set to 5000 by default)

GET https://infinite-api.tcgplayer.com/priceguide/set/1356/cards/?rows=5000

response

each result item

Code Sample

import org.json.JSONArray;
import org.json.JSONObject;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;

public class TCGPlayerAPIExample {
    public static void main(String[] args) {
        try {
            // Specify the URL for the GET request
            String url = "https://infinite-api.tcgplayer.com/priceguide/set/1356/cards/?rows=5000";

            // Create a URL object
            URL apiUrl = new URL(url);

            // Open a connection to the URL
            HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();

            // Set the request method to GET
            connection.setRequestMethod("GET");

            // Get the response code
            int responseCode = connection.getResponseCode();

            // Check if the request was successful (status code 200)
            if (responseCode == HttpURLConnection.HTTP_OK) {
                // Read the response from the API
                BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                String inputLine;
                StringBuilder response = new StringBuilder();

                while ((inputLine = in.readLine()) != null) {
                    response.append(inputLine);
                }
                in.close();

                // Parse the JSON response
                JSONObject jsonResponse = new JSONObject(response.toString());

                // Extract information from the JSON response
                JSONArray cards = jsonResponse.getJSONArray("results");
                for (int i = 0; i < cards.length(); i++) {
                    JSONObject card = cards.getJSONObject(i);
                    String condition = card.getString("condition");
                    double productID = card.getString("productID");
                    double productConditionID = card.getDouble("productConditionID");
                    String game = card.getString("game");
                    double lowPrice = card.getString("lowPrice");
                    double marketPrice = card.getString("marketPrice");
                    String number = card.getString("number");
                    String printing = card.getString("printing");
                    String productName = card.getString("productName");
                    String rarity = card.getString("rarity");
                    double sales = card.getString("sales");
                    String set = card.getString("set");
                    String setAbbrv = card.getString("setAbbrv");
                    String type = card.getString("type");
                    
                    // Process or print the extracted information as needed
                    System.out.println("Card Name: " + cardName);
                    System.out.println("Card Price: " + productID );
                    System.out.println("-----------");
                }
            } else {
                // If the request was not successful, print the error code
                System.out.println("GET request failed. Response Code: " + responseCode);
            }

            // Close the connection
            connection.disconnect();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Note: Code not tested

Answered By - Dean Van Greunen