Tutorial: When and how to use the official TikTok API

# package import
import requests
import json
import time
import pandas as pd
from tqdm.auto import tqdm

### Set your client key and secret from your dev account
client_key = "<YOUR_KEY>"
client_secret = "YOUR_SECRET"
## api base_url
base_url = 'https://open.tiktokapis.com/v2/oauth/token/'
# set header and payload info for API request
headers = {'Content-Type': 'application/x-www-form-urlencoded', 
        'Cache-Control': 'no-cache'}
payload = {"client_key":client_key,
        "client_secret":client_secret,
        "grant_type":"client_credentials"}
# request bearer auth token
response = requests.post(base_url, headers=headers, data=payload)
# get data from response
response_as_json = response.json()
# save access token as a variable for further usage
access_token = response_as_json["access_token"]

usernames = ["tagesschau", "zeitimbild","20minuten"]

# Define API-Endpoint URL
# The Endpoint-URL also defines the variables we will collect about the respective videos
# We include all currently available variables
url = "https://open.tiktokapis.com/v2/research/video/query/?fields=id,video_description,create_time,region_code,share_count,view_count,like_count,comment_count,music_id,hashtag_names,username,effect_ids,playlist_id,voice_to_text"
# With the header, we verify ourselves with our access token and indicate the format of the data we want to scrape
headers = {
    "Authorization": f"Bearer {access_token}",
    "Content-Type": "application/json"
}
# We also need to set additional query parameters:
# First, the start and end date on which the videos have been uploaded
# "YYYYMMDD" format as string, both are meant inclusive, no more than 30 days time frame possible
# We will look at a workaround to the 30-day rule in a minute.
start_date = "20240805"
end_date = "20240901"
# The default number of items returned by a request is 20.
# We set it to a maximum of 100 entries.
max_count = 100

def scrape_videos_userbased(username, max_count, start_date, end_date, headers):
    # We ask the API: Give us all videos uploaded between start_date and end_date whose uploader is in our username lists
    query_params = {
                    "query": {
                        "and": [
                            {"operation": "EQ", "field_name": "username", "field_values": [username]}
                        ]
                    },
                    "max_count": max_count,
                    "start_date": start_date,
                    "end_date": end_date
                    }
    # Make the call and transform the response to a JSON file
    response = requests.post(url, headers=headers, data=json.dumps(query_params))
    r = response.json()
    # return the response parsed as a JSON file
    return r

usernames = ["20minuten"]
url = "https://open.tiktokapis.com/v2/research/video/query/?fields=id,video_description,create_time,region_code,share_count,view_count,like_count,comment_count,music_id,hashtag_names,username,effect_ids,playlist_id,voice_to_text"
headers = {
    "Authorization": f"Bearer {access_token}",
    "Content-Type": "application/json"
}
start_date = "20240805"
end_date = "20240901"
max_count = 100 

# We add the search_id and cursor as input variables as well as to the query.
def scrape_videos_pagination(username, max_count, start_date, end_date, headers, search_id, cursor):
    query_params = {
                    "query": {
                        "and": [
                            {"operation": "EQ", "field_name": "username", "field_values": [username]}
                        ]
                    },
                    "fields": "id,video_description,create_time",
                    "max_count": max_count,
                    "start_date": start_date,
                    "end_date": end_date,
                    "cursor": cursor,
                    "search_id":search_id
                    }
    # Make the call and transform the response to a JSON file
    response = requests.post(url, headers=headers, data=json.dumps(query_params))
    r = response.json()
    # return the response parsed as a JSON file
    return r

from datetime import timedelta
from datetime import datetime
def generate_date_range(start_date, end_date): 
    '''
    function to generate a list of date sections to facilitate the scraping of 
    longer time periods beyon the nativ 30 days limit of the API.
    input: desired start and end date in "YYYY-MM-DD" format.
    both are meant inclusive
    '''
    
    start = datetime.strptime(start_date, "%Y%m%d")
    end = datetime.strptime(end_date, "%Y%m%d")
    delta = end - start
    current_date = start
    
    dates = []
    temp_end_date = current_date
    while current_date <= end:
        delta_end = end-current_date
        if delta_end.days <= 30:
            # if the distance to actual end is less than 30 days break while loop
            dates.append((current_date.strftime("%Y-%m-%d").replace("-",""), end.strftime("%Y-%m-%d").replace("-","")))
            break
        else:
            # Move to the next date, 30 days later
            temp_end_date += timedelta(days=29)
            dates.append((current_date.strftime("%Y-%m-%d").replace("-",""),
                        temp_end_date.strftime("%Y-%m-%d").replace("-","")))
            current_date = temp_end_date + timedelta(days=1)
        
    return dates

url = "https://open.tiktokapis.com/v2/research/video/query/?fields=id,video_description,create_time,region_code,share_count,view_count,like_count,comment_count,music_id,hashtag_names,username,effect_ids,playlist_id,voice_to_text"
headers = {
    "Authorization": f"Bearer {access_token}",
    "Content-Type": "application/json"
}
max_count = 100
usernames = ["20minuten"]